1 module CustomerTokenTest; 2 3 import RuleTranslatorLexer; 4 import RuleTranslatorParser: RuleTranslatorParser; 5 import RuleTranslatorBaseListener; 6 import antlr.v4.runtime.ANTLRInputStream; 7 import antlr.v4.runtime.CommonToken; 8 import antlr.v4.runtime.CommonTokenFactory; 9 import antlr.v4.runtime.TokenFactory; 10 import antlr.v4.runtime.CommonTokenStream; 11 import antlr.v4.runtime.TokenStream; 12 import antlr.v4.runtime.CharStream; 13 import antlr.v4.runtime.DiagnosticErrorListener; 14 import antlr.v4.runtime.LexerNoViableAltException; 15 import antlr.v4.runtime.Token; 16 import antlr.v4.runtime.TokenSource; 17 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter; 18 import antlr.v4.runtime.atn.ParserATNSimulator; 19 import antlr.v4.runtime.tree.ParseTreeWalker; 20 import dshould : be, equal, not, should; 21 import dshould.thrown; 22 import std.conv : to; 23 import std.stdio : File, writefln; 24 import std.file; 25 import unit_threaded; 26 import std.typecons; 27 import std.variant; 28 29 class ResultTokenFactory : CommonTokenFactory { 30 CharStream input; 31 32 this(CharStream input) { 33 this.input = input; 34 } 35 36 override 37 ResultToken create(int type, Variant text) { 38 return new ResultToken(type, text); 39 } 40 41 override 42 ResultToken create(TokenFactorySourcePair source, int type, 43 Variant text, int channel, int start, int stop, 44 int line, int charPositionInLine ) { 45 auto t = new ResultToken(source, type, channel, 46 start, stop); 47 t.setLine(line); 48 t.setCharPositionInLine(charPositionInLine); 49 t.srcName = input.getSourceName; 50 return t; 51 } 52 } 53 54 /** 55 * A Token source with Result as additional attribute 56 */ 57 58 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 59 60 struct Result { ushort indent; string text;} 61 Result[] res; 62 63 ushort indent; 64 65 class ResultToken : CommonToken { 66 67 public string srcName; 68 69 this (int type, Variant text) { 70 super(type, text); 71 } 72 73 this( TokenFactorySourcePair source, int type, 74 int channel, int start, int stop) { 75 super(source, type, channel, start, stop); 76 } 77 78 override string toString() { 79 return srcName ~ ":" ~ super.toString; 80 } 81 82 override Variant getText() { 83 return super.getText; 84 } 85 } 86 87 public class InsertTestListenerDelete : RuleTranslatorBaseListener { 88 89 TokenStreamRewriter rewriter; 90 91 this(TokenStream tokens) 92 { 93 rewriter = new TokenStreamRewriter(tokens); 94 } 95 /** 96 * <p>The default implementation does nothing.</p> 97 */ 98 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 99 } 100 101 /** 102 * {@inheritDoc} 103 * 104 * <p>The default implementation does nothing.</p> 105 */ 106 override public void exitStmt(RuleTranslatorParser.StmtContext ctx) { 107 debug(TokenStreamRewriter) { 108 import std.stdio : writefln; 109 writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop); 110 } 111 rewriter.deleteT(ctx.start, ctx.stop); 112 } 113 } 114 115 public class ResultListener : RuleTranslatorBaseListener { 116 117 TokenStreamRewriter rewriter; 118 119 this(TokenStream tokens) 120 { 121 rewriter = new TokenStreamRewriter(tokens); 122 } 123 /** 124 * <p>The default implementation does nothing.</p> 125 */ 126 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 127 } 128 129 /** 130 * {@inheritDoc} 131 * 132 * <p>The default implementation does nothing.</p> 133 */ 134 override public void exitStmt(RuleTranslatorParser.StmtContext ctx) { 135 debug(TokenStreamRewriter) { 136 import std.stdio : writefln; 137 writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop); 138 } 139 rewriter.deleteT(ctx.start, ctx.stop); 140 } 141 } 142 143 @Tags("CustomerToken") 144 @("add source name") 145 unittest { 146 147 auto expected = 148 ` 149 ruleDelayasDELAYde 150 basede.Phrases 151 `; 152 auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r")); 153 auto lexer = new RuleTranslatorLexer(antlrInput); 154 auto factory = new ResultTokenFactory(antlrInput); 155 lexer.tokenFactory(factory); 156 auto ln = lexer.getGrammarFileName; 157 ln.should.equal("RuleTranslator.g4"); 158 auto cts = new CommonTokenStream(lexer); 159 cts.fill; 160 auto f = File("unittest/complex/customer_tokens.cmp"); 161 auto charRange = f.byLine(); 162 string s; 163 int i; 164 foreach (t; charRange) { 165 s = cts.get(i++).to!string; 166 s.should.equal(t); 167 } 168 f.close; 169 auto parser = new RuleTranslatorParser(cts); 170 parser.tokenFactory(factory); 171 // Specify entry point 172 auto rootContext = parser.file_input; 173 parser.numberOfSyntaxErrors.should.be.equal(0); 174 auto extractor = new InsertTestListenerDelete(cts); 175 auto walker = new ParseTreeWalker; 176 walker.walk(extractor, rootContext); 177 auto str = extractor.rewriter.getText.get!(string); 178 str.should.equal(expected); 179 } 180