1 module RuleTokenTest;
2 
3 import RuleTranslatorBaseListener : RuleTranslatorBaseListener;
4 import RuleTranslatorLexer : RuleTranslatorLexer;
5 import RuleTranslatorParser: RuleTranslatorParser;
6 import antlr.v4.runtime.ANTLRInputStream;
7 import antlr.v4.runtime.CommonToken;
8 import antlr.v4.runtime.CommonTokenFactory;
9 import antlr.v4.runtime.TokenFactory;
10 import antlr.v4.runtime.CommonTokenStream;
11 import antlr.v4.runtime.TokenStream;
12 import antlr.v4.runtime.tree.TerminalNode;
13 import antlr.v4.runtime.CharStream;
14 import antlr.v4.runtime.Token;
15 import antlr.v4.runtime.TokenSource;
16 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter;
17 import antlr.v4.runtime.atn.ParserATNSimulator;
18 import antlr.v4.runtime.tree.ParseTreeWalker;
19 import dshould;
20 import std.conv : to;
21 import std.file;
22 import std.stdio : File, writefln;
23 import std.typecons;
24 import std.variant;
25 
26 class ResultTokenFactory : CommonTokenFactory {
27 
28     this(CharStream input) {
29     }
30 
31     override
32     ResultToken create(int type, Variant text) {
33         return new ResultToken(type, text);
34     }
35 
36     override
37     ResultToken create(TokenFactorySourcePair source, int type,
38                        Variant text, int channel, size_t start, size_t stop,
39                        int line, int charPositionInLine ) {
40         auto t = new  ResultToken(source, type, channel,
41                                   start, stop);
42         t.setLine(line);
43         t.setCharPositionInLine(charPositionInLine);
44         return t;
45     }
46 }
47 
48 /**
49  * A Token source with Result as additional attribute
50  */
51 
52 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
53 
54 struct Result { ushort indent; string text;}
55 
56 ushort indent = 1; // 0 reserved for space
57 
58 class ResultToken : CommonToken {
59 
60     public Result[] res;
61 
62     this (int type, Variant text) {
63         super(type, text);
64     }
65 
66     this( TokenFactorySourcePair source, int type,
67           int channel, size_t start, size_t stop) {
68         super(source, type, channel, start, stop);
69         Result r;
70         r.indent = 1;
71         r.text = super.getText.to!string;
72         res ~= r;
73     }
74 
75     override string toString() {
76         import std.format : format;
77         return format("%s", res);
78     }
79 
80     override Variant getText() {
81         Variant r = res;
82         return r;
83     }
84 }
85 
86 public class ResultListener : RuleTranslatorBaseListener {
87 
88     private TokenStreamRewriter rewriter;
89 
90     private bool nextOfNewline = false;
91 
92     ushort indentText = 1;
93 
94     this(TokenStream tokens)
95     {
96         rewriter = new TokenStreamRewriter(tokens);
97     }
98     /**
99      * <p>The default implementation does nothing.</p>
100      */
101     override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) {
102     }
103 
104     /**
105      *
106      * <p>The default implementation does nothing.</p>
107      */
108     override public void enterFunct_stmt(RuleTranslatorParser.Funct_stmtContext ctx) {
109         Result[] newText;
110         Result r;
111         r. indent = 2;
112         r. text = `zug_function_number 3`;
113         newText ~= r;
114         r. indent = 3;
115         r. text = `"von"`;
116         newText ~= r;
117         r. indent = 2;
118         r. text = `"nach"`;
119         newText ~= r;
120         Variant v = newText;
121         rewriter.replace(ctx.start, ctx.stop, v);
122     }
123 
124     /**
125      * {@inheritDoc}
126      *
127      * <p>The default implementation does nothing.</p>
128      */
129     override public void visitTerminal(TerminalNode node) {
130         debug (TokenStreamRewriter) {
131             writefln("\nvisitTerminal -> \"%s\", node.symbol -> %s, indentText = %s",
132                      node.getText,
133                      node.getSymbol,
134                      indentText
135                      );
136         }
137         switch (node.getSymbol.getType) {
138 
139         case RuleTranslatorParser.INDENT:
140             indentText++;
141             break;
142         case RuleTranslatorParser.DEDENT:
143             indentText--;
144             auto r = node.getText.get!(Result[]);
145             r[0].text = "";
146             break;
147         case RuleTranslatorParser.NEWLINE:
148             nextOfNewline = true;
149             import std.array : join;
150             Result r;
151             r.indent = indentText;
152             break;
153         case RuleTranslatorParser.EOF:
154             debug(TokenStreamRewriter) {
155                 import std.stdio : writeln;
156                 writeln("\n+++ EOF +++");
157             }
158             break;
159         default:
160             debug(TokenStreamRewriter)
161                 writefln("\ndefault: node.getText.type = %s", node.getText.type);
162             if (nextOfNewline) {
163                 auto r = node.getText.get!(Result[]);
164                 r[0].indent = indentText;
165             }
166             nextOfNewline = false;
167             break;
168         }
169     }
170 
171 }
172 
173 @("using result struct")
174 unittest
175 {
176     auto expected =
177         `rule Delay as DELAY de
178 base de . Phrases
179 if a :
180     "Information" "zu"
181     "Zug"
182     zug_function_number 3
183         "von"
184     "nach"
185 "Berlin"
186 `;
187     auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r"));
188     auto lexer = new RuleTranslatorLexer(antlrInput);
189     auto factory = new ResultTokenFactory(antlrInput);
190     lexer.tokenFactory(factory);
191     auto ln = lexer.getGrammarFileName;
192     auto cts = new CommonTokenStream(lexer);
193     cts.fill;
194     auto parser = new RuleTranslatorParser(cts);
195     parser.tokenFactory(factory);
196     // Specify entry point
197     auto rootContext = parser.file_input;
198     parser.numberOfSyntaxErrors.should.be.equal(0);
199     auto extractor = new ResultListener(cts);
200     auto walker = new ParseTreeWalker;
201     walker.walk(extractor, rootContext);
202 
203     import std.array : appender;
204     auto buf = appender!(string);
205 
206     import std.stdio;
207 
208     auto r = extractor.rewriter.getText.get!(Result[]);
209     bool firstOnLine = true;
210 
211     foreach (i, el; r)
212         {
213             import std..string : strip;
214             if (!i)
215                 continue;
216 
217             if (el.text == "\n") {
218                 firstOnLine = true;
219                 buf.put("\n");
220                 continue;
221             }
222 
223             if (el.indent >1) {
224                 buf.put("\n");
225                 while (--el.indent)
226                     buf.put("    ");
227                 buf.put(el.text);
228             }
229             else {
230                 if (firstOnLine) {
231                     if (el.text.strip.length) {
232                         buf.put(el.text);
233                     }
234                     else {
235                         // DEDENT
236                         firstOnLine = true;
237                         continue;
238                     }
239                     firstOnLine = false;
240                 }
241                 else {
242                     if (el.text.strip.length)
243                         buf.put(" " ~ el.text.strip);
244                 }
245             }
246         }
247     buf.data.should.equal(expected);
248 }