1 module RuleTokenTest; 2 3 import RuleTranslatorBaseListener : RuleTranslatorBaseListener; 4 import RuleTranslatorLexer : RuleTranslatorLexer; 5 import RuleTranslatorParser: RuleTranslatorParser; 6 import antlr.v4.runtime.ANTLRInputStream; 7 import antlr.v4.runtime.CommonToken; 8 import antlr.v4.runtime.CommonTokenFactory; 9 import antlr.v4.runtime.TokenFactory; 10 import antlr.v4.runtime.CommonTokenStream; 11 import antlr.v4.runtime.TokenStream; 12 import antlr.v4.runtime.tree.TerminalNode; 13 import antlr.v4.runtime.CharStream; 14 import antlr.v4.runtime.Token; 15 import antlr.v4.runtime.TokenSource; 16 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter; 17 import antlr.v4.runtime.atn.ParserATNSimulator; 18 import antlr.v4.runtime.tree.ParseTreeWalker; 19 import dshould; 20 import std.conv : to; 21 import std.file; 22 import std.stdio : File, writefln; 23 import std.typecons; 24 import std.variant; 25 26 class ResultTokenFactory : CommonTokenFactory { 27 28 this(CharStream input) { 29 } 30 31 override 32 ResultToken create(int type, Variant text) { 33 return new ResultToken(type, text); 34 } 35 36 override 37 ResultToken create(TokenFactorySourcePair source, int type, 38 Variant text, int channel, size_t start, size_t stop, 39 int line, int charPositionInLine ) { 40 auto t = new ResultToken(source, type, channel, 41 start, stop); 42 t.setLine(line); 43 t.setCharPositionInLine(charPositionInLine); 44 return t; 45 } 46 } 47 48 /** 49 * A Token source with Result as additional attribute 50 */ 51 52 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 53 54 struct Result { ushort indent; string text;} 55 56 ushort indent = 1; // 0 reserved for space 57 58 class ResultToken : CommonToken { 59 60 public Result[] res; 61 62 this (int type, Variant text) { 63 super(type, text); 64 } 65 66 this( TokenFactorySourcePair source, int type, 67 int channel, size_t start, size_t stop) { 68 super(source, type, channel, start, stop); 69 Result r; 70 r.indent = 1; 71 r.text = super.getText.to!string; 72 res ~= r; 73 } 74 75 override string toString() { 76 import std.format : format; 77 return format("%s", res); 78 } 79 80 override Variant getText() { 81 Variant r = res; 82 return r; 83 } 84 } 85 86 public class ResultListener : RuleTranslatorBaseListener { 87 88 private TokenStreamRewriter rewriter; 89 90 private bool nextOfNewline = false; 91 92 ushort indentText = 1; 93 94 this(TokenStream tokens) 95 { 96 rewriter = new TokenStreamRewriter(tokens); 97 } 98 /** 99 * <p>The default implementation does nothing.</p> 100 */ 101 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 102 } 103 104 /** 105 * 106 * <p>The default implementation does nothing.</p> 107 */ 108 override public void enterFunct_stmt(RuleTranslatorParser.Funct_stmtContext ctx) { 109 Result[] newText; 110 Result r; 111 r. indent = 2; 112 r. text = `zug_function_number 3`; 113 newText ~= r; 114 r. indent = 3; 115 r. text = `"von"`; 116 newText ~= r; 117 r. indent = 2; 118 r. text = `"nach"`; 119 newText ~= r; 120 Variant v = newText; 121 rewriter.replace(ctx.start, ctx.stop, v); 122 } 123 124 /** 125 * {@inheritDoc} 126 * 127 * <p>The default implementation does nothing.</p> 128 */ 129 override public void visitTerminal(TerminalNode node) { 130 debug (TokenStreamRewriter) { 131 writefln("\nvisitTerminal -> \"%s\", node.symbol -> %s, indentText = %s", 132 node.getText, 133 node.getSymbol, 134 indentText 135 ); 136 } 137 switch (node.getSymbol.getType) { 138 139 case RuleTranslatorParser.INDENT: 140 indentText++; 141 break; 142 case RuleTranslatorParser.DEDENT: 143 indentText--; 144 auto r = node.getText.get!(Result[]); 145 r[0].text = ""; 146 break; 147 case RuleTranslatorParser.NEWLINE: 148 nextOfNewline = true; 149 import std.array : join; 150 Result r; 151 r.indent = indentText; 152 break; 153 case RuleTranslatorParser.EOF: 154 debug(TokenStreamRewriter) { 155 import std.stdio : writeln; 156 writeln("\n+++ EOF +++"); 157 } 158 break; 159 default: 160 debug(TokenStreamRewriter) 161 writefln("\ndefault: node.getText.type = %s", node.getText.type); 162 if (nextOfNewline) { 163 auto r = node.getText.get!(Result[]); 164 r[0].indent = indentText; 165 } 166 nextOfNewline = false; 167 break; 168 } 169 } 170 171 } 172 173 @("using result struct") 174 unittest 175 { 176 auto expected = 177 `rule Delay as DELAY de 178 base de . Phrases 179 if a : 180 "Information" "zu" 181 "Zug" 182 zug_function_number 3 183 "von" 184 "nach" 185 "Berlin" 186 `; 187 auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r")); 188 auto lexer = new RuleTranslatorLexer(antlrInput); 189 auto factory = new ResultTokenFactory(antlrInput); 190 lexer.tokenFactory(factory); 191 auto ln = lexer.getGrammarFileName; 192 auto cts = new CommonTokenStream(lexer); 193 cts.fill; 194 auto parser = new RuleTranslatorParser(cts); 195 parser.tokenFactory(factory); 196 // Specify entry point 197 auto rootContext = parser.file_input; 198 parser.numberOfSyntaxErrors.should.be.equal(0); 199 auto extractor = new ResultListener(cts); 200 auto walker = new ParseTreeWalker; 201 walker.walk(extractor, rootContext); 202 203 import std.array : appender; 204 auto buf = appender!(string); 205 206 import std.stdio; 207 208 auto r = extractor.rewriter.getText.get!(Result[]); 209 bool firstOnLine = true; 210 211 foreach (i, el; r) 212 { 213 import std..string : strip; 214 if (!i) 215 continue; 216 217 if (el.text == "\n") { 218 firstOnLine = true; 219 buf.put("\n"); 220 continue; 221 } 222 223 if (el.indent >1) { 224 buf.put("\n"); 225 while (--el.indent) 226 buf.put(" "); 227 buf.put(el.text); 228 } 229 else { 230 if (firstOnLine) { 231 if (el.text.strip.length) { 232 buf.put(el.text); 233 } 234 else { 235 // DEDENT 236 firstOnLine = true; 237 continue; 238 } 239 firstOnLine = false; 240 } 241 else { 242 if (el.text.strip.length) 243 buf.put(" " ~ el.text.strip); 244 } 245 } 246 } 247 buf.data.should.equal(expected); 248 }