1 module CustomerTokenTest; 2 3 import RuleTranslatorLexer; 4 import RuleTranslatorParser: RuleTranslatorParser; 5 import RuleTranslatorBaseListener; 6 import antlr.v4.runtime.ANTLRInputStream; 7 import antlr.v4.runtime.CommonToken; 8 import antlr.v4.runtime.CommonTokenFactory; 9 import antlr.v4.runtime.TokenFactory; 10 import antlr.v4.runtime.CommonTokenStream; 11 import antlr.v4.runtime.TokenStream; 12 import antlr.v4.runtime.CharStream; 13 import antlr.v4.runtime.DiagnosticErrorListener; 14 import antlr.v4.runtime.LexerNoViableAltException; 15 import antlr.v4.runtime.Token; 16 import antlr.v4.runtime.TokenSource; 17 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter; 18 import antlr.v4.runtime.atn.ParserATNSimulator; 19 import antlr.v4.runtime.tree.ParseTreeWalker; 20 import dshould : be, equal, not, should; 21 import dshould.thrown; 22 import std.conv : to; 23 import std.stdio : File, writefln; 24 import std.file; 25 import unit_threaded; 26 import std.typecons; 27 import std.variant; 28 29 class ResultTokenFactory : CommonTokenFactory { 30 CharStream input; 31 32 this(CharStream input) { 33 this.input = input; 34 } 35 36 override 37 ResultToken create(int type, Variant text) { 38 return new ResultToken(type, text); 39 } 40 41 override 42 ResultToken create(TokenFactorySourcePair source, int type, 43 Variant text, int channel, int start, int stop, 44 int line, int charPositionInLine ) { 45 auto t = new ResultToken(source, type, channel, 46 start, stop); 47 t.setLine(line); 48 t.setCharPositionInLine(charPositionInLine); 49 t.srcName = input.getSourceName; 50 return t; 51 } 52 } 53 54 /** 55 * A Token source with Result as additional attribute 56 */ 57 58 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 59 60 struct Result { ushort indent; string text;} 61 Result[] res; 62 63 ushort indent; 64 65 class ResultToken : CommonToken { 66 67 public string srcName; 68 69 this (int type, Variant text) { 70 super(type, text); 71 } 72 73 this( TokenFactorySourcePair source, int type, 74 int channel, int start, int stop) { 75 super(source, type, channel, start, stop); 76 } 77 78 override string toString() { 79 return srcName ~ ":" ~ super.toString; 80 } 81 82 override Variant getText() { 83 import std.stdio; 84 writefln("\ngetText: super.getText = %s", super.getText); 85 return super.getText; 86 } 87 } 88 89 public class InsertTestListenerDelete : RuleTranslatorBaseListener { 90 91 TokenStreamRewriter rewriter; 92 93 this(TokenStream tokens) 94 { 95 rewriter = new TokenStreamRewriter(tokens); 96 } 97 /** 98 * <p>The default implementation does nothing.</p> 99 */ 100 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 101 } 102 103 /** 104 * {@inheritDoc} 105 * 106 * <p>The default implementation does nothing.</p> 107 */ 108 override public void exitStmt(RuleTranslatorParser.StmtContext ctx) { 109 debug(TokenStreamRewriter) { 110 import std.stdio : writefln; 111 writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop); 112 } 113 rewriter.deleteT(ctx.start, ctx.stop); 114 } 115 } 116 117 public class ResultListener : RuleTranslatorBaseListener { 118 119 TokenStreamRewriter rewriter; 120 121 this(TokenStream tokens) 122 { 123 rewriter = new TokenStreamRewriter(tokens); 124 } 125 /** 126 * <p>The default implementation does nothing.</p> 127 */ 128 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 129 } 130 131 /** 132 * {@inheritDoc} 133 * 134 * <p>The default implementation does nothing.</p> 135 */ 136 override public void exitStmt(RuleTranslatorParser.StmtContext ctx) { 137 debug(TokenStreamRewriter) { 138 import std.stdio : writefln; 139 writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop); 140 } 141 rewriter.deleteT(ctx.start, ctx.stop); 142 } 143 } 144 145 @Tags("CustomerToken") 146 @("add source name") 147 unittest { 148 149 auto expected = 150 ` 151 ruleDelayasDELAYde 152 basede.Phrases 153 `; 154 auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r")); 155 auto lexer = new RuleTranslatorLexer(antlrInput); 156 auto factory = new ResultTokenFactory(antlrInput); 157 lexer.tokenFactory(factory); 158 auto ln = lexer.getGrammarFileName; 159 auto cts = new CommonTokenStream(lexer); 160 cts.fill; 161 auto f = File("unittest/complex/customer_tokens.cmp"); 162 auto charRange = f.byLine(); 163 string s; 164 int i; 165 foreach (t; charRange) { 166 s = cts.get(i++).to!string; 167 s.should.equal(t); 168 } 169 f.close; 170 auto parser = new RuleTranslatorParser(cts); 171 parser.tokenFactory(factory); 172 // Specify entry point 173 auto rootContext = parser.file_input; 174 parser.numberOfSyntaxErrors.should.be.equal(0); 175 auto extractor = new InsertTestListenerDelete(cts); 176 auto walker = new ParseTreeWalker; 177 walker.walk(extractor, rootContext); 178 auto str = extractor.rewriter.getText.get!(string); 179 str.should.equal(expected); 180 } 181