1 module CustomerTokenTest;
2 
3 import RuleTranslatorLexer;
4 import RuleTranslatorParser: RuleTranslatorParser;
5 import RuleTranslatorBaseListener;
6 import antlr.v4.runtime.ANTLRInputStream;
7 import antlr.v4.runtime.CommonToken;
8 import antlr.v4.runtime.CommonTokenFactory;
9 import antlr.v4.runtime.TokenFactory;
10 import antlr.v4.runtime.CommonTokenStream;
11 import antlr.v4.runtime.TokenStream;
12 import antlr.v4.runtime.CharStream;
13 import antlr.v4.runtime.DiagnosticErrorListener;
14 import antlr.v4.runtime.LexerNoViableAltException;
15 import antlr.v4.runtime.Token;
16 import antlr.v4.runtime.TokenSource;
17 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter;
18 import antlr.v4.runtime.atn.ParserATNSimulator;
19 import antlr.v4.runtime.tree.ParseTreeWalker;
20 import dshould : be, equal, not, should;
21 import dshould.thrown;
22 import std.conv : to;
23 import std.stdio : File, writefln;
24 import std.file;
25 import unit_threaded;
26 import std.typecons;
27 import std.variant;
28 
29 class ResultTokenFactory : CommonTokenFactory {
30     CharStream input;
31 
32     this(CharStream input) {
33         this.input = input;
34     }
35 
36     override
37     ResultToken create(int type, Variant text) {
38         return new ResultToken(type, text);
39     }
40 
41     override
42     ResultToken create(TokenFactorySourcePair source, int type,
43                        Variant text, int channel, int start, int stop,
44                        int line, int charPositionInLine ) {
45         auto t = new  ResultToken(source, type, channel,
46                                   start, stop);
47         t.setLine(line);
48         t.setCharPositionInLine(charPositionInLine);
49         t.srcName = input.getSourceName;
50         return t;
51     }
52 }
53 
54 /**
55  * A Token source with Result as additional attribute
56  */
57 
58 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
59 
60 struct Result { ushort indent; string text;}
61 Result[] res;
62 
63 ushort indent;
64 
65 class ResultToken : CommonToken {
66 
67     public string srcName;
68 
69     this (int type, Variant text) {
70         super(type, text);
71     }
72 
73     this( TokenFactorySourcePair source, int type,
74           int channel, int start, int stop) {
75         super(source, type, channel, start, stop);
76     }
77 
78     override string toString() {
79         return srcName ~ ":" ~ super.toString;
80     }
81 
82     override Variant getText() {
83         import std.stdio;
84         writefln("\ngetText: super.getText = %s", super.getText);
85         return super.getText;
86     }
87 }
88 
89 public class InsertTestListenerDelete : RuleTranslatorBaseListener {
90 
91     TokenStreamRewriter rewriter;
92 
93     this(TokenStream tokens)
94     {
95         rewriter = new TokenStreamRewriter(tokens);
96     }
97     /**
98      * <p>The default implementation does nothing.</p>
99      */
100     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
101     }
102 
103     /**
104      * {@inheritDoc}
105      *
106      * <p>The default implementation does nothing.</p>
107      */
108     override public void exitStmt(RuleTranslatorParser.StmtContext ctx) {
109         debug(TokenStreamRewriter) {
110             import std.stdio : writefln;
111             writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop);
112         }
113         rewriter.deleteT(ctx.start, ctx.stop);
114     }
115 }
116 
117 public class ResultListener : RuleTranslatorBaseListener {
118 
119     TokenStreamRewriter rewriter;
120 
121     this(TokenStream tokens)
122     {
123         rewriter = new TokenStreamRewriter(tokens);
124     }
125     /**
126      * <p>The default implementation does nothing.</p>
127      */
128     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
129     }
130 
131     /**
132      * {@inheritDoc}
133      *
134      * <p>The default implementation does nothing.</p>
135      */
136     override public void exitStmt(RuleTranslatorParser.StmtContext ctx) {
137         debug(TokenStreamRewriter) {
138             import std.stdio : writefln;
139             writefln("exitStmt ctx.start = %s, ctx.stop = %s", ctx.start, ctx.stop);
140         }
141         rewriter.deleteT(ctx.start, ctx.stop);
142     }
143 }
144 
145 @Tags("CustomerToken")
146 @("add source name")
147 unittest {
148 
149     auto expected =
150         `
151 ruleDelayasDELAYde
152 basede.Phrases
153 `;
154     auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r"));
155     auto lexer = new RuleTranslatorLexer(antlrInput);
156     auto factory = new ResultTokenFactory(antlrInput);
157     lexer.tokenFactory(factory);
158     auto ln = lexer.getGrammarFileName;
159     auto cts = new CommonTokenStream(lexer);
160     cts.fill;
161     auto f = File("unittest/complex/customer_tokens.cmp");
162     auto charRange = f.byLine();
163     string s;
164     int i;
165     foreach (t; charRange) {
166         s = cts.get(i++).to!string;
167         s.should.equal(t);
168     }
169     f.close;
170     auto parser = new RuleTranslatorParser(cts);
171     parser.tokenFactory(factory);
172     // Specify entry point
173     auto rootContext = parser.file_input;
174     parser.numberOfSyntaxErrors.should.be.equal(0);
175     auto extractor = new InsertTestListenerDelete(cts);
176     auto walker = new ParseTreeWalker;
177     walker.walk(extractor, rootContext);
178     auto str = extractor.rewriter.getText.get!(string);
179     str.should.equal(expected);
180 }
181