1 module CustomerTokenTest;
2 
3 import RuleTranslatorLexer;
4 import RuleTranslatorParser: RuleTranslatorParser;
5 import RuleTranslatorBaseListener;
6 import antlr.v4.runtime.ANTLRInputStream;
7 import antlr.v4.runtime.CommonToken;
8 import antlr.v4.runtime.CommonTokenFactory;
9 import antlr.v4.runtime.TokenFactory;
10 import antlr.v4.runtime.CommonTokenStream;
11 import antlr.v4.runtime.TokenStream;
12 import antlr.v4.runtime.CharStream;
13 import antlr.v4.runtime.DiagnosticErrorListener;
14 import antlr.v4.runtime.LexerNoViableAltException;
15 import antlr.v4.runtime.Token;
16 import antlr.v4.runtime.TokenSource;
17 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter;
18 import antlr.v4.runtime.atn.ParserATNSimulator;
19 import antlr.v4.runtime.tree.ParseTreeWalker;
20 import dshould : be, equal, not, should;
21 import dshould.thrown;
22 import std.conv : to;
23 import std.stdio : File, writefln;
24 import std.file;
25 import unit_threaded;
26 import std.typecons;
27 import std.variant;
28 
29 class ResultTokenFactory : CommonTokenFactory {
30     CharStream input;
31 
32     this(CharStream input) {
33         this.input = input;
34     }
35 
36     override
37     ResultToken create(int type, Variant text) {
38         return new ResultToken(type, text);
39     }
40 
41     override
42     ResultToken create(TokenFactorySourcePair source, int type,
43                        Variant text, int channel, size_t start, size_t stop,
44                        int line, int charPositionInLine ) {
45         auto t = new  ResultToken(source, type, channel,
46                                   start, stop);
47         t.setLine(line);
48         t.setCharPositionInLine(charPositionInLine);
49         t.srcName = input.getSourceName;
50         return t;
51     }
52 }
53 
54 /**
55  * A Token source with Result as additional attribute
56  */
57 
58 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
59 
60 struct Result { ushort indent; string text;}
61 Result[] res;
62 
63 ushort indent;
64 
65 class ResultToken : CommonToken {
66 
67     public string srcName;
68 
69     this (int type, Variant text) {
70         super(type, text);
71     }
72 
73     this( TokenFactorySourcePair source, int type,
74           int channel, size_t start, size_t stop) {
75         super(source, type, channel, start, stop);
76     }
77 
78     override string toString() {
79         return srcName ~ ":" ~ super.toString;
80     }
81 
82     override Variant getText() {
83         return super.getText;
84     }
85 }
86 
87 public class InsertTestListenerDelete : RuleTranslatorBaseListener {
88 
89     TokenStreamRewriter rewriter;
90 
91     this(TokenStream tokens)
92     {
93         rewriter = new TokenStreamRewriter(tokens);
94     }
95     /**
96      * <p>The default implementation does nothing.</p>
97      */
98     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
99     }
100 
101     /**
102      * {@inheritDoc}
103      *
104      * <p>The default implementation does nothing.</p>
105      */
106     override public void exitStmt(RuleTranslatorParser.StmtContext ctx) {
107         debug(TokenStreamRewriter) {
108             import std.stdio : writefln;
109             writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop);
110         }
111         rewriter.deleteT(ctx.start, ctx.stop);
112     }
113 }
114 
115 public class ResultListener : RuleTranslatorBaseListener {
116 
117     TokenStreamRewriter rewriter;
118 
119     this(TokenStream tokens)
120     {
121         rewriter = new TokenStreamRewriter(tokens);
122     }
123     /**
124      * <p>The default implementation does nothing.</p>
125      */
126     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
127     }
128 
129     /**
130      * {@inheritDoc}
131      *
132      * <p>The default implementation does nothing.</p>
133      */
134     override public void exitStmt(RuleTranslatorParser.StmtContext ctx) {
135         debug(TokenStreamRewriter) {
136             import std.stdio : writefln;
137             writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop);
138         }
139         rewriter.deleteT(ctx.start, ctx.stop);
140     }
141 }
142 
143 @Tags("CustomerToken")
144 @("add source name")
145 unittest {
146 
147     auto expected =
148         `
149 ruleDelayasDELAYde
150 basede.Phrases
151 `;
152     auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r"));
153     auto lexer = new RuleTranslatorLexer(antlrInput);
154     auto factory = new ResultTokenFactory(antlrInput);
155     lexer.tokenFactory(factory);
156     auto ln = lexer.getGrammarFileName;
157     ln.should.equal("RuleTranslator.g4");
158     auto cts = new CommonTokenStream(lexer);
159     cts.fill;
160     auto f = File("unittest/complex/customer_tokens.cmp");
161     auto charRange = f.byLine();
162     string s;
163     int i;
164     foreach (t; charRange) {
165         s = cts.get(i++).to!string;
166         s.should.equal(t);
167     }
168     f.close;
169     auto parser = new RuleTranslatorParser(cts);
170     parser.tokenFactory(factory);
171     // Specify entry point
172     auto rootContext = parser.file_input;
173     parser.numberOfSyntaxErrors.should.be.equal(0);
174     auto extractor = new InsertTestListenerDelete(cts);
175     auto walker = new ParseTreeWalker;
176     walker.walk(extractor, rootContext);
177     auto str = extractor.rewriter.getText.get!(string);
178     str.should.equal(expected);
179 }
180