1 module CustomerTokenTest;
2 
3 import RuleTranslatorBaseListener : RuleTranslatorBaseListener;
4 import RuleTranslatorLexer : RuleTranslatorLexer;
5 import RuleTranslatorParser: RuleTranslatorParser;
6 import antlr.v4.runtime.ANTLRInputStream;
7 import antlr.v4.runtime.CommonToken;
8 import antlr.v4.runtime.CommonTokenFactory;
9 import antlr.v4.runtime.TokenFactory;
10 import antlr.v4.runtime.CommonTokenStream;
11 import antlr.v4.runtime.TokenStream;
12 import antlr.v4.runtime.CharStream;
13 import antlr.v4.runtime.DiagnosticErrorListener;
14 import antlr.v4.runtime.LexerNoViableAltException;
15 import antlr.v4.runtime.Token;
16 import antlr.v4.runtime.TokenSource;
17 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter;
18 import antlr.v4.runtime.atn.ParserATNSimulator;
19 import antlr.v4.runtime.tree.ParseTreeWalker;
20 import dshould;
21 import std.conv : to;
22 import std.file;
23 import std.stdio : File, writefln;
24 import std.typecons;
25 import std.variant;
26 
27 class ResultTokenFactory : CommonTokenFactory {
28     CharStream input;
29 
30     this(CharStream input) {
31         this.input = input;
32     }
33 
34     override
35     ResultToken create(int type, Variant text) {
36         return new ResultToken(type, text);
37     }
38 
39     override
40     ResultToken create(TokenFactorySourcePair source, int type,
41                        Variant text, int channel, size_t start, size_t stop,
42                        int line, int charPositionInLine ) {
43         auto t = new  ResultToken(source, type, channel,
44                                   start, stop);
45         t.setLine(line);
46         t.setCharPositionInLine(charPositionInLine);
47         t.srcName = input.getSourceName;
48         return t;
49     }
50 }
51 
52 /**
53  * A Token source with Result as additional attribute
54  */
55 
56 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
57 
58 struct Result { ushort indent; string text;}
59 Result[] res;
60 
61 ushort indent;
62 
63 class ResultToken : CommonToken {
64 
65     public string srcName;
66 
67     this (int type, Variant text) {
68         super(type, text);
69     }
70 
71     this( TokenFactorySourcePair source, int type,
72           int channel, size_t start, size_t stop) {
73         super(source, type, channel, start, stop);
74     }
75 
76     override string toString() {
77         return srcName ~ ":" ~ super.toString;
78     }
79 
80     override Variant getText() {
81         return super.getText;
82     }
83 }
84 
85 public class InsertTestListenerDelete : RuleTranslatorBaseListener {
86 
87     TokenStreamRewriter rewriter;
88 
89     this(TokenStream tokens)
90     {
91         rewriter = new TokenStreamRewriter(tokens);
92     }
93     /**
94      * <p>The default implementation does nothing.</p>
95      */
96     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
97     }
98 
99     /**
100      * {@inheritDoc}
101      *
102      * <p>The default implementation does nothing.</p>
103      */
104     override public void exitStmt(RuleTranslatorParser.StmtContext ctx) {
105         debug(TokenStreamRewriter) {
106             import std.stdio : writefln;
107             writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop);
108         }
109         rewriter.deleteT(ctx.start, ctx.stop);
110     }
111 }
112 
113 public class ResultListener : RuleTranslatorBaseListener {
114 
115     TokenStreamRewriter rewriter;
116 
117     this(TokenStream tokens)
118     {
119         rewriter = new TokenStreamRewriter(tokens);
120     }
121     /**
122      * <p>The default implementation does nothing.</p>
123      */
124     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
125     }
126 
127     /**
128      * {@inheritDoc}
129      *
130      * <p>The default implementation does nothing.</p>
131      */
132     override public void exitStmt(RuleTranslatorParser.StmtContext ctx) {
133         debug(TokenStreamRewriter) {
134             import std.stdio : writefln;
135             writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop);
136         }
137         rewriter.deleteT(ctx.start, ctx.stop);
138     }
139 }
140 
141 @("add source name")
142 unittest
143 {
144     auto expected =
145         `
146 ruleDelayasDELAYde
147 basede.Phrases
148 `;
149     auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r"));
150     auto lexer = new RuleTranslatorLexer(antlrInput);
151     auto factory = new ResultTokenFactory(antlrInput);
152     lexer.tokenFactory(factory);
153     auto ln = lexer.getGrammarFileName;
154     ln.should.equal("RuleTranslator.g4");
155     auto cts = new CommonTokenStream(lexer);
156     cts.fill;
157     auto f = File("unittest/complex/customer_tokens.cmp");
158     auto charRange = f.byLine();
159     string s;
160     int i;
161     foreach (t; charRange) {
162         s = cts.get(i++).to!string;
163         s.should.equal(t);
164     }
165     f.close;
166     auto parser = new RuleTranslatorParser(cts);
167     parser.tokenFactory(factory);
168     // Specify entry point
169     auto rootContext = parser.file_input;
170     parser.numberOfSyntaxErrors.should.be.equal(0);
171     auto extractor = new InsertTestListenerDelete(cts);
172     auto walker = new ParseTreeWalker;
173     walker.walk(extractor, rootContext);
174     auto str = extractor.rewriter.getText.get!(string);
175     str.should.equal(expected);
176 }