1 module RuleTokenTest;
2 
3 import RuleTranslatorLexer;
4 import RuleTranslatorParser: RuleTranslatorParser;
5 import RuleTranslatorBaseListener;
6 import antlr.v4.runtime.ANTLRInputStream;
7 import antlr.v4.runtime.CommonToken;
8 import antlr.v4.runtime.CommonTokenFactory;
9 import antlr.v4.runtime.TokenFactory;
10 import antlr.v4.runtime.CommonTokenStream;
11 import antlr.v4.runtime.TokenStream;
12 import antlr.v4.runtime.tree.TerminalNode;
13 import antlr.v4.runtime.CharStream;
14 import antlr.v4.runtime.Token;
15 import antlr.v4.runtime.TokenSource;
16 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter;
17 import antlr.v4.runtime.atn.ParserATNSimulator;
18 import antlr.v4.runtime.tree.ParseTreeWalker;
19 import dshould : be, equal, not, should;
20 import dshould.thrown;
21 import std.conv : to;
22 import std.stdio : File, writefln;
23 import std.file;
24 import unit_threaded;
25 import std.typecons;
26 import std.variant;
27 
28 class ResultTokenFactory : CommonTokenFactory {
29 
30     this(CharStream input) {
31     }
32 
33     override
34     ResultToken create(int type, Variant text) {
35         return new ResultToken(type, text);
36     }
37 
38     override
39     ResultToken create(TokenFactorySourcePair source, int type,
40                        Variant text, int channel, int start, int stop,
41                        int line, int charPositionInLine ) {
42         auto t = new  ResultToken(source, type, channel,
43                                   start, stop);
44         t.setLine(line);
45         t.setCharPositionInLine(charPositionInLine);
46         return t;
47     }
48 }
49 
50 /**
51  * A Token source with Result as additional attribute
52  */
53 
54 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
55 
56 struct Result { ushort indent; string text;}
57 
58 ushort indent = 1; // 0 reserved for space
59 
60 class ResultToken : CommonToken {
61 
62     public Result[] res;
63 
64     this (int type, Variant text) {
65         super(type, text);
66     }
67 
68     this( TokenFactorySourcePair source, int type,
69           int channel, int start, int stop) {
70         super(source, type, channel, start, stop);
71         Result r;
72         r.indent = 1;
73         r.text = super.getText.to!string;
74         res ~= r;
75     }
76 
77     override string toString() {
78         import std.format : format;
79         return format("%s", res);
80     }
81 
82     override Variant getText() {
83         Variant r = res;
84         return r;
85     }
86 }
87 
88 public class ResultListener : RuleTranslatorBaseListener {
89 
90     private TokenStreamRewriter rewriter;
91 
92     private bool nextOfNewline = false;
93 
94     ushort indentText = 1;
95 
96     this(TokenStream tokens)
97     {
98         rewriter = new TokenStreamRewriter(tokens);
99     }
100     /**
101      * <p>The default implementation does nothing.</p>
102      */
103     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
104     }
105 
106     /**
107      *
108      * <p>The default implementation does nothing.</p>
109      */
110     override public void enterFunct_stmt(RuleTranslatorParser.Funct_stmtContext ctx) {
111         Result[] newText;
112         Result r;
113         r. indent = 2;
114         r. text = `zug_function_number 3`;
115         newText ~= r;
116         r. indent = 3;
117         r. text = `"von"`;
118         newText ~= r;
119         r. indent = 2;
120         r. text = `"nach"`;
121         newText ~= r;
122         Variant v = newText;
123         rewriter.replace(ctx.start, ctx.stop, v);
124     }
125 
126     /**
127      * {@inheritDoc}
128      *
129      * <p>The default implementation does nothing.</p>
130      */
131     override public void visitTerminal(TerminalNode node) {
132         debug (TokenStreamRewriter) {
133             writefln("\nvisitTerminal -> \"%s\", node.symbol -> %s, indentText = %s",
134                      node.getText,
135                      node.getSymbol,
136                      indentText
137                      );
138         }
139         switch (node.getSymbol.getType) {
140 
141         case RuleTranslatorParser.INDENT:
142             indentText++;
143             break;
144         case RuleTranslatorParser.DEDENT:
145             indentText--;
146             auto r = node.getText.get!(Result[]);
147             r[0].text = "";
148             break;
149         case RuleTranslatorParser.NEWLINE:
150             nextOfNewline = true;
151             import std.array : join;
152             Result r;
153             r.indent = indentText;
154             break;
155         case RuleTranslatorParser.EOF:
156             debug(TokenStreamRewriter) {
157                 import std.stdio : writeln;
158                 writeln("\n+++ EOF +++");
159             }
160             break;
161         default:
162             debug(TokenStreamRewriter)
163                 writefln("\ndefault: node.getText.type = %s", node.getText.type);
164             if (nextOfNewline) {
165                 auto r = node.getText.get!(Result[]);
166                 r[0].indent = indentText;
167             }
168             nextOfNewline = false;
169             break;
170         }
171     }
172 
173 }
174 
175 @Tags("CustomerToken", "tt")
176 @("using result struct")
177 unittest {
178 
179     auto expected =
180         `rule Delay as DELAY de
181 base de . Phrases
182 if a :
183     "Information" "zu"
184     "Zug"
185     zug_function_number 3
186         "von"
187     "nach"
188 "Berlin"
189 `;
190     auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r"));
191     auto lexer = new RuleTranslatorLexer(antlrInput);
192     auto factory = new ResultTokenFactory(antlrInput);
193     lexer.tokenFactory(factory);
194     auto ln = lexer.getGrammarFileName;
195     auto cts = new CommonTokenStream(lexer);
196     cts.fill;
197     auto parser = new RuleTranslatorParser(cts);
198     parser.tokenFactory(factory);
199     // Specify entry point
200     auto rootContext = parser.file_input;
201     parser.numberOfSyntaxErrors.should.be.equal(0);
202     auto extractor = new ResultListener(cts);
203     auto walker = new ParseTreeWalker;
204     walker.walk(extractor, rootContext);
205 
206     import std.array : appender;
207     auto buf = appender!(string);
208 
209     import std.stdio;
210 
211     auto r = extractor.rewriter.getText.get!(Result[]);
212     bool firstOnLine = true;
213 
214     foreach (i, el; r)
215         {
216             import std.string : strip;
217             if (!i)
218                 continue;
219 
220             if (el.text == "\n") {
221                 firstOnLine = true;
222                 buf.put("\n");
223                 continue;
224             }
225 
226             if (el.indent >1) {
227                 buf.put("\n");
228                 while (--el.indent)
229                     buf.put("    ");
230                 buf.put(el.text);
231             }
232             else {
233                 if (firstOnLine) {
234                     if (el.text.strip.length) {
235                         buf.put(el.text);
236                     }
237                     else {
238                         // DEDENT
239                         firstOnLine = true;
240                         continue;
241                     }
242                     firstOnLine = false;
243                 }
244                 else {
245                     if (el.text.strip.length)
246                         buf.put(" " ~ el.text.strip);
247                 }
248             }
249         }
250     buf.data.should.equal(expected);
251 }