1 module RuleTokenTest; 2 3 import RuleTranslatorLexer; 4 import RuleTranslatorParser: RuleTranslatorParser; 5 import RuleTranslatorBaseListener; 6 import antlr.v4.runtime.ANTLRInputStream; 7 import antlr.v4.runtime.CommonToken; 8 import antlr.v4.runtime.CommonTokenFactory; 9 import antlr.v4.runtime.TokenFactory; 10 import antlr.v4.runtime.CommonTokenStream; 11 import antlr.v4.runtime.TokenStream; 12 import antlr.v4.runtime.tree.TerminalNode; 13 import antlr.v4.runtime.CharStream; 14 import antlr.v4.runtime.Token; 15 import antlr.v4.runtime.TokenSource; 16 import antlr.v4.runtime.TokenStreamRewriter : TokenStreamRewriter; 17 import antlr.v4.runtime.atn.ParserATNSimulator; 18 import antlr.v4.runtime.tree.ParseTreeWalker; 19 import dshould : be, equal, not, should; 20 import dshould.thrown; 21 import std.conv : to; 22 import std.stdio : File, writefln; 23 import std.file; 24 import unit_threaded; 25 import std.typecons; 26 import std.variant; 27 28 class ResultTokenFactory : CommonTokenFactory { 29 30 this(CharStream input) { 31 } 32 33 override 34 ResultToken create(int type, Variant text) { 35 return new ResultToken(type, text); 36 } 37 38 override 39 ResultToken create(TokenFactorySourcePair source, int type, 40 Variant text, int channel, int start, int stop, 41 int line, int charPositionInLine ) { 42 auto t = new ResultToken(source, type, channel, 43 start, stop); 44 t.setLine(line); 45 t.setCharPositionInLine(charPositionInLine); 46 return t; 47 } 48 } 49 50 /** 51 * A Token source with Result as additional attribute 52 */ 53 54 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 55 56 struct Result { ushort indent; string text;} 57 58 ushort indent = 1; // 0 reserved for space 59 60 class ResultToken : CommonToken { 61 62 public Result[] res; 63 64 this (int type, Variant text) { 65 super(type, text); 66 } 67 68 this( TokenFactorySourcePair source, int type, 69 int channel, int start, int stop) { 70 super(source, type, channel, start, stop); 71 Result r; 72 r.indent = 1; 73 r.text = super.getText.to!string; 74 res ~= r; 75 } 76 77 override string toString() { 78 import std.format : format; 79 return format("%s", res); 80 } 81 82 override Variant getText() { 83 Variant r = res; 84 return r; 85 } 86 } 87 88 public class ResultListener : RuleTranslatorBaseListener { 89 90 private TokenStreamRewriter rewriter; 91 92 private bool nextOfNewline = false; 93 94 ushort indentText = 1; 95 96 this(TokenStream tokens) 97 { 98 rewriter = new TokenStreamRewriter(tokens); 99 } 100 /** 101 * <p>The default implementation does nothing.</p> 102 */ 103 override public void enterFile_input(RuleTranslatorParser.File_inputContext ctx) { 104 } 105 106 /** 107 * 108 * <p>The default implementation does nothing.</p> 109 */ 110 override public void enterFunct_stmt(RuleTranslatorParser.Funct_stmtContext ctx) { 111 Result[] newText; 112 Result r; 113 r. indent = 2; 114 r. text = `zug_function_number 3`; 115 newText ~= r; 116 r. indent = 3; 117 r. text = `"von"`; 118 newText ~= r; 119 r. indent = 2; 120 r. text = `"nach"`; 121 newText ~= r; 122 Variant v = newText; 123 rewriter.replace(ctx.start, ctx.stop, v); 124 } 125 126 /** 127 * {@inheritDoc} 128 * 129 * <p>The default implementation does nothing.</p> 130 */ 131 override public void visitTerminal(TerminalNode node) { 132 debug (TokenStreamRewriter) { 133 writefln("\nvisitTerminal -> \"%s\", node.symbol -> %s, indentText = %s", 134 node.getText, 135 node.getSymbol, 136 indentText 137 ); 138 } 139 switch (node.getSymbol.getType) { 140 141 case RuleTranslatorParser.INDENT: 142 indentText++; 143 break; 144 case RuleTranslatorParser.DEDENT: 145 indentText--; 146 auto r = node.getText.get!(Result[]); 147 r[0].text = ""; 148 break; 149 case RuleTranslatorParser.NEWLINE: 150 nextOfNewline = true; 151 import std.array : join; 152 Result r; 153 r.indent = indentText; 154 break; 155 case RuleTranslatorParser.EOF: 156 debug(TokenStreamRewriter) { 157 import std.stdio : writeln; 158 writeln("\n+++ EOF +++"); 159 } 160 break; 161 default: 162 debug(TokenStreamRewriter) 163 writefln("\ndefault: node.getText.type = %s", node.getText.type); 164 if (nextOfNewline) { 165 auto r = node.getText.get!(Result[]); 166 r[0].indent = indentText; 167 } 168 nextOfNewline = false; 169 break; 170 } 171 } 172 173 } 174 175 @Tags("CustomerToken", "tt") 176 @("using result struct") 177 unittest { 178 179 auto expected = 180 `rule Delay as DELAY de 181 base de . Phrases 182 if a : 183 "Information" "zu" 184 "Zug" 185 zug_function_number 3 186 "von" 187 "nach" 188 "Berlin" 189 `; 190 auto antlrInput = new ANTLRInputStream(File("unittest/complex/rule.rul", "r")); 191 auto lexer = new RuleTranslatorLexer(antlrInput); 192 auto factory = new ResultTokenFactory(antlrInput); 193 lexer.tokenFactory(factory); 194 auto ln = lexer.getGrammarFileName; 195 auto cts = new CommonTokenStream(lexer); 196 cts.fill; 197 auto parser = new RuleTranslatorParser(cts); 198 parser.tokenFactory(factory); 199 // Specify entry point 200 auto rootContext = parser.file_input; 201 parser.numberOfSyntaxErrors.should.be.equal(0); 202 auto extractor = new ResultListener(cts); 203 auto walker = new ParseTreeWalker; 204 walker.walk(extractor, rootContext); 205 206 import std.array : appender; 207 auto buf = appender!(string); 208 209 import std.stdio; 210 211 auto r = extractor.rewriter.getText.get!(Result[]); 212 bool firstOnLine = true; 213 214 foreach (i, el; r) 215 { 216 import std.string : strip; 217 if (!i) 218 continue; 219 220 if (el.text == "\n") { 221 firstOnLine = true; 222 buf.put("\n"); 223 continue; 224 } 225 226 if (el.indent >1) { 227 buf.put("\n"); 228 while (--el.indent) 229 buf.put(" "); 230 buf.put(el.text); 231 } 232 else { 233 if (firstOnLine) { 234 if (el.text.strip.length) { 235 buf.put(el.text); 236 } 237 else { 238 // DEDENT 239 firstOnLine = true; 240 continue; 241 } 242 firstOnLine = false; 243 } 244 else { 245 if (el.text.strip.length) 246 buf.put(" " ~ el.text.strip); 247 } 248 } 249 } 250 buf.data.should.equal(expected); 251 }