1 module CustomerTokenTest; 2 3 import RuleTranslatorBaseListener : RuleTranslatorBaseListener; 4 import RuleTranslatorLexer : RuleTranslatorLexer; 5 import RuleTranslatorParser: RuleTranslatorParser; 6 import antlr.v4.runtime.ANTLRInputStream; 7 import antlr.v4.runtime.CommonToken; 8 import antlr.v4.runtime.CommonTokenFactory; 9 import antlr.v4.runtime.TokenFactory; 10 import antlr.v4.runtime.CommonTokenStream; 11 import antlr.v4.runtime.TokenStream; 12 import antlr.v4.runtime.CharStream; 13 import antlr.v4.runtime.DiagnosticErrorListener; 14 import antlr.v4.runtime.LexerNoViableAltException; 15 import antlr.v4.runtime.Token; 16 import antlr.v4.runtime.TokenSource; 17 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter; 18 import antlr.v4.runtime.atn.ParserATNSimulator; 19 import antlr.v4.runtime.tree.ParseTreeWalker; 20 import dshould; 21 import std.conv : to; 22 import std.file; 23 import std.stdio : File, writefln; 24 import std.typecons; 25 import std.variant; 26 27 class ResultTokenFactory : CommonTokenFactory { 28 CharStream input; 29 30 this(CharStream input) { 31 this.input = input; 32 } 33 34 override 35 ResultToken create(int type, Variant text) { 36 return new ResultToken(type, text); 37 } 38 39 override 40 ResultToken create(TokenFactorySourcePair source, int type, 41 Variant text, int channel, size_t start, size_t stop, 42 int line, int charPositionInLine ) { 43 auto t = new ResultToken(source, type, channel, 44 start, stop); 45 t.setLine(line); 46 t.setCharPositionInLine(charPositionInLine); 47 t.srcName = input.getSourceName; 48 return t; 49 } 50 } 51 52 /** 53 * A Token source with Result as additional attribute 54 */ 55 56 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 57 58 struct Result { ushort indent; string text;} 59 Result[] res; 60 61 ushort indent; 62 63 class ResultToken : CommonToken { 64 65 public string srcName; 66 67 this (int type, Variant text) { 68 super(type, text); 69 } 70 71 this( TokenFactorySourcePair source, int type, 72 int channel, size_t start, size_t stop) { 73 super(source, type, channel, start, stop); 74 } 75 76 override string toString() { 77 return srcName ~ ":" ~ super.toString; 78 } 79 80 override Variant getText() { 81 return super.getText; 82 } 83 } 84 85 public class InsertTestListenerDelete : RuleTranslatorBaseListener { 86 87 TokenStreamRewriter rewriter; 88 89 this(TokenStream tokens) 90 { 91 rewriter = new TokenStreamRewriter(tokens); 92 } 93 /** 94 * <p>The default implementation does nothing.</p> 95 */ 96 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 97 } 98 99 /** 100 * {@inheritDoc} 101 * 102 * <p>The default implementation does nothing.</p> 103 */ 104 override public void exitStmt(RuleTranslatorParser.StmtContext ctx) { 105 debug(TokenStreamRewriter) { 106 import std.stdio : writefln; 107 writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop); 108 } 109 rewriter.deleteT(ctx.start, ctx.stop); 110 } 111 } 112 113 public class ResultListener : RuleTranslatorBaseListener { 114 115 TokenStreamRewriter rewriter; 116 117 this(TokenStream tokens) 118 { 119 rewriter = new TokenStreamRewriter(tokens); 120 } 121 /** 122 * <p>The default implementation does nothing.</p> 123 */ 124 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 125 } 126 127 /** 128 * {@inheritDoc} 129 * 130 * <p>The default implementation does nothing.</p> 131 */ 132 override public void exitStmt(RuleTranslatorParser.StmtContext ctx) { 133 debug(TokenStreamRewriter) { 134 import std.stdio : writefln; 135 writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop); 136 } 137 rewriter.deleteT(ctx.start, ctx.stop); 138 } 139 } 140 141 @("add source name") 142 unittest 143 { 144 auto expected = 145 ` 146 ruleDelayasDELAYde 147 basede.Phrases 148 `; 149 auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r")); 150 auto lexer = new RuleTranslatorLexer(antlrInput); 151 auto factory = new ResultTokenFactory(antlrInput); 152 lexer.tokenFactory(factory); 153 auto ln = lexer.getGrammarFileName; 154 ln.should.equal("RuleTranslator.g4"); 155 auto cts = new CommonTokenStream(lexer); 156 cts.fill; 157 auto f = File("unittest/complex/customer_tokens.cmp"); 158 auto charRange = f.byLine(); 159 string s; 160 int i; 161 foreach (t; charRange) { 162 s = cts.get(i++).to!string; 163 s.should.equal(t); 164 } 165 f.close; 166 auto parser = new RuleTranslatorParser(cts); 167 parser.tokenFactory(factory); 168 // Specify entry point 169 auto rootContext = parser.file_input; 170 parser.numberOfSyntaxErrors.should.be.equal(0); 171 auto extractor = new InsertTestListenerDelete(cts); 172 auto walker = new ParseTreeWalker; 173 walker.walk(extractor, rootContext); 174 auto str = extractor.rewriter.getText.get!(string); 175 str.should.equal(expected); 176 }