1 /*
2  * Copyright (c) 2012-2020 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.atn.LexerATNSimulator;
8 
9 import std.conv;
10 import std.format;
11 import std.stdio;
12 import antlr.v4.runtime.atn.ATNSimulator;
13 import antlr.v4.runtime.IntStreamConstant;
14 import antlr.v4.runtime.Lexer;
15 import antlr.v4.runtime.UnsupportedOperationException;
16 import antlr.v4.runtime.dfa.DFA;
17 import antlr.v4.runtime.dfa.DFAState;
18 import antlr.v4.runtime.atn.ATN;
19 import antlr.v4.runtime.atn.ATNState;
20 import antlr.v4.runtime.atn.ATNConfigSet;
21 import antlr.v4.runtime.atn.ATNConfig;
22 import antlr.v4.runtime.atn.LexerActionExecutor;
23 import antlr.v4.runtime.atn.LexerATNConfig;
24 import antlr.v4.runtime.atn.PredictionContext;
25 import antlr.v4.runtime.atn.RuleTransition;
26 import antlr.v4.runtime.atn.RuleStopState;
27 import antlr.v4.runtime.atn.PredictionContextCache;
28 import antlr.v4.runtime.atn.SimState;
29 import antlr.v4.runtime.atn.SingletonPredictionContext;
30 import antlr.v4.runtime.atn.PredicateTransition;
31 import antlr.v4.runtime.atn.Transition;
32 import antlr.v4.runtime.atn.TransitionStates;
33 import antlr.v4.runtime.atn.ActionTransition;
34 import antlr.v4.runtime.atn.OrderedATNConfigSet;
35 import antlr.v4.runtime.CharStream;
36 import antlr.v4.runtime.LexerNoViableAltException;
37 import antlr.v4.runtime.Token;
38 import antlr.v4.runtime.TokenConstantDefinition;
39 import antlr.v4.runtime.misc;
40 
41 /**
42  * "dup" of ParserInterpreter
43  */
44 class LexerATNSimulator : ATNSimulator
45 {
46 
47     public static immutable int MIN_DFA_EDGE = 0;
48 
49     /**
50      * forces unicode to stay in ATN
51      */
52     public static immutable int MAX_DFA_EDGE = 127;
53 
54     protected Lexer recog;
55 
56     /**
57      * @uml
58      * The current token's starting index into the character stream.
59      * Shared across DFA to ATN simulation in case the ATN fails and the
60      * DFA did not have a previous accept state. In this case, we use the
61      * ATN-generated exception object.
62      */
63     protected int startIndex = -1;
64 
65     /**
66      * @uml
67      * line number 1..n within the input
68      */
69     protected int line = 1;
70 
71     /**
72      * @uml
73      * The index of the character relative to the beginning of the line 0..n-1
74      */
75     protected int charPositionInLine = 0;
76 
77     public DFA[] decisionToDFA;
78 
79     protected int mode = Lexer.DEFAULT_MODE;
80 
81     protected SimState prevAccept;
82 
83     public static int match_calls = 0;
84 
85     public this(ATN atn, DFA[] decisionToDFA, PredictionContextCache sharedContextCache)
86     {
87         this(null, atn, decisionToDFA,sharedContextCache);
88     }
89 
90     public this(Lexer recog, ATN atn, DFA[] decisionToDFA, PredictionContextCache sharedContextCache)
91     {
92         super(atn, sharedContextCache);
93         this.decisionToDFA = decisionToDFA;
94         this.recog = recog;
95     }
96 
97     public void copyState(LexerATNSimulator simulator)
98     {
99         this.charPositionInLine = simulator.charPositionInLine;
100         this.line = simulator.line;
101         this.mode = simulator.mode;
102         this.startIndex = simulator.startIndex;
103     }
104 
105     public int match(CharStream input, int mode)
106     {
107         match_calls++;
108         this.mode = mode;
109         int mark = input.mark;
110         try {
111             this.startIndex = to!int(input.index);
112             this.prevAccept.reset;
113             DFA dfa = decisionToDFA[mode];
114             if (dfa.s0 is null) {
115                 return matchATN(input);
116             }
117             else {
118                 return execATN(input, dfa.s0);
119             }
120         }
121         finally {
122             input.release(mark);
123         }
124     }
125 
126     /**
127      * @uml
128      * @override
129      */
130     public override void reset()
131     {
132         prevAccept.reset();
133         startIndex = -1;
134         line = 1;
135         charPositionInLine = 0;
136         mode = Lexer.DEFAULT_MODE;
137     }
138 
139     /**
140      * @uml
141      * @override
142      */
143     public override void clearDFA()
144     {
145         for (int d = 0; d < decisionToDFA.length; d++) {
146             decisionToDFA[d] = new DFA(atn.getDecisionState(d), d);
147         }
148     }
149 
150     protected int matchATN(CharStream input)
151     {
152         ATNState startState = atn.modeToStartState[mode];
153         debug(LexerATNSimulator)
154             writefln!"matchATN mode %s start: %s"(mode, startState);
155         int old_mode = mode;
156         ATNConfigSet s0_closure = computeStartState(input, startState);
157         bool suppressEdge = s0_closure.hasSemanticContext;
158         s0_closure.hasSemanticContext = false;
159         DFAState next = addDFAState(s0_closure);
160         if (!suppressEdge) {
161             decisionToDFA[mode].s0 = next;
162         }
163         int predict = execATN(input, next);
164 
165         debug(LexerATNSimulator)
166             writefln("DFA after matchATN:\n%1$s\n", decisionToDFA[old_mode].toLexerString());
167         return predict;
168     }
169 
170     protected int execATN(CharStream input, DFAState ds0)
171     {
172         debug(LexerATNSimulator) {
173             writefln("start state closure %s from %s", input.index, ds0.configs);
174         }
175 
176         if (ds0.isAcceptState) {
177             // allow zero-length tokens
178             captureSimState(prevAccept, input, ds0);
179         }
180         auto t = input.LA(1);
181         DFAState s = ds0; // s is current/from DFA state
182 
183         while (true) { // while more work
184             debug(LexerATNSimulator) {
185                 writefln("execATN loop starting closure: %s\n", s.configs);
186             }
187             // As we move src->trg, src->trg, we keep track of the previous trg to
188             // avoid looking up the DFA state again, which is expensive.
189             // If the previous target was already part of the DFA, we might
190             // be able to avoid doing a reach operation upon t. If s!=null,
191             // it means that semantic predicates didn't prevent us from
192             // creating a DFA state. Once we know s!=null, we check to see if
193             // the DFA state has an edge already for t. If so, we can just reuse
194             // it's configuration set; there's no point in re-computing it.
195             // This is kind of like doing DFA simulation within the ATN
196             // simulation because DFA simulation is really just a way to avoid
197             // computing reach/closure sets. Technically, once we know that
198             // we have a previously added DFA state, we could jump over to
199             // the DFA simulator. But, that would mean popping back and forth
200             // a lot and making things more complicated algorithmically.
201             // This optimization makes a lot of sense for loops within DFA.
202             // A character will take us back to an existing DFA state
203             // that already has lots of edges out of it. e.g., .* in comments.
204             DFAState target = getExistingTargetState(s, t);
205             if (!target) {
206                 target = computeTargetState(input, s, t);
207             }
208             if (target is ERROR) {
209                 break;
210             }
211             // If this is a consumable input element, make sure to consume before
212             // capturing the accept state so the input index, line, and char
213             // position accurately reflect the state of the interpreter at the
214             // end of the token.
215             if (t != IntStreamConstant.EOF) {
216                 consume(input);
217             }
218 
219             if (target.isAcceptState) {
220                 captureSimState(prevAccept, input, target);
221                 if (t == IntStreamConstant.EOF) {
222                     break;
223                 }
224             }
225             t = input.LA(1);
226             s = target; // flip; current DFA target becomes new src/from state
227         }
228         return failOrAccept(prevAccept, input, s.configs, t);
229     }
230 
231     /**
232      * @uml
233      * Get an existing target state for an edge in the DFA. If the target state
234      * for the edge has not yet been computed or is otherwise not available,
235      * this method returns {@code null}.
236      *
237      *  @param s The current DFA state
238      *  @param t The next input symbol
239      *  @return The existing target DFA state for the given input symbol
240      * {@code t}, or {@code null} if the target state for this edge is not
241      * already cached
242      */
243     public DFAState getExistingTargetState(DFAState s, int t)
244     {
245         if (s.edges is null || t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) {
246             return null;
247         }
248 
249         DFAState target = s.edges[t - MIN_DFA_EDGE];
250         if (target !is null)
251             debug(LexerATNSimulator) {
252                 writefln("reuse state %1$s"~
253                          " edge to %2$s", s.stateNumber, target.stateNumber);
254             }
255 
256         return target;
257     }
258 
259     /**
260      * @uml
261      * Compute a target state for an edge in the DFA, and attempt to add the
262      * computed state and corresponding edge to the DFA.
263      *
264      *  @param input The input stream
265      *  @param s The current DFA state
266      *  @param t The next input symbol
267      *
268      *  @return The computed target DFA state for the given input symbol
269      * {@code t}. If {@code t} does not lead to a valid DFA state, this method
270      * returns {@link #ERROR}.
271      */
272     protected DFAState computeTargetState(CharStream input, DFAState s, dchar t)
273     {
274         ATNConfigSet reach = new OrderedATNConfigSet();
275 
276         // if we don't find an existing DFA state
277         // Fill reach starting from closure, following t transitions
278         getReachableConfigSet(input, s.configs, reach, t);
279 
280         if (reach.isEmpty()) { // we got nowhere on t from s
281             if (!reach.hasSemanticContext) {
282                 // we got nowhere on t, don't throw out this knowledge; it'd
283                 // cause a failover from DFA later.
284                 addDFAEdge(s, t, ERROR);
285             }
286 
287             // stop when we can't match any more char
288             return ERROR;
289         }
290 
291         // Add an edge from s to target DFA found/created for reach
292         return addDFAEdge(s, t, reach);
293     }
294 
295     protected int failOrAccept(SimState prevAccept, CharStream input, ATNConfigSet reach,
296                                dchar t)
297     {
298         if (prevAccept.dfaState) {
299             LexerActionExecutor lexerActionExecutor = prevAccept.dfaState.lexerActionExecutor;
300             accept(input, lexerActionExecutor, startIndex,
301                    prevAccept.index, prevAccept.line, prevAccept.charPos);
302             return prevAccept.dfaState.prediction;
303         }
304         else {
305             // if no accept and EOF is first char, return EOF
306             if (t == IntStreamConstant.EOF && input.index == startIndex) {
307                 return TokenConstantDefinition.EOF;
308             }
309             throw new LexerNoViableAltException(recog, input, startIndex, reach);
310         }
311     }
312 
313     /**
314      * Given a starting configuration set, figure out all ATN configurations
315      * we can reach upon input {@code t}. Parameter {@code reach} is a return
316      * parameter.
317      */
318     protected void getReachableConfigSet(CharStream input, ATNConfigSet closureATNConfigSet, ATNConfigSet reach,
319                                          dchar t)
320     {
321         // this is used to skip processing for configs which have a lower priority
322         // than a config that already reached an accept state for the same rule
323         int skipAlt = ATN.INVALID_ALT_NUMBER;
324         foreach (ATNConfig c; closureATNConfigSet.configs) {
325             bool currentAltReachedAcceptState = c.alt == skipAlt;
326             if (currentAltReachedAcceptState && (cast(LexerATNConfig)c).hasPassedThroughNonGreedyDecision()) {
327                 continue;
328             }
329 
330             debug(LexerATNSimulator) {
331                 writefln("testing %s at %s\n", getTokenName(t), c.toString(recog, true));
332             }
333 
334             int n = c.state.getNumberOfTransitions();
335             for (int ti=0; ti<n; ti++) {               // for each transition
336                 Transition trans = c.state.transition(ti);
337                 ATNState target = getReachableTarget(trans, t);
338                 if (target) {
339                     LexerActionExecutor lexerActionExecutor = (cast(LexerATNConfig)c).getLexerActionExecutor();
340                     if (lexerActionExecutor) {
341                         lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex);
342                     }
343 
344                     bool treatEofAsEpsilon = t == IntStreamConstant.EOF;
345                     if (closure(input, new LexerATNConfig(cast(LexerATNConfig)c, target, lexerActionExecutor),
346                                 reach, currentAltReachedAcceptState, true, treatEofAsEpsilon)) {
347                         // any remaining configs for this alt have a lower priority than
348                         // the one that just reached an accept state.
349                         skipAlt = c.alt;
350                         break;
351                     }
352                 }
353             }
354         }
355 
356     }
357 
358     protected void accept(CharStream input, LexerActionExecutor lexerActionExecutor, size_t startIndex,
359                           size_t index, int line, int charPos)
360     {
361         debug(LexerATNSimulator) {
362             writefln("ACTION %s\n", lexerActionExecutor);
363         }
364 
365         // seek to after last char in token
366         input.seek(index);
367         this.line = line;
368         this.charPositionInLine = charPos;
369 
370         if (lexerActionExecutor !is null && recog !is null) {
371             lexerActionExecutor.execute(recog, input, startIndex);
372         }
373     }
374 
375     protected ATNState getReachableTarget(Transition trans, dchar t)
376     {
377         if (trans.matches(t, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) {
378             return trans.target;
379         }
380         return null;
381     }
382 
383     protected ATNConfigSet computeStartState(CharStream input, ATNState p)
384     {
385         PredictionContext initialContext = cast(PredictionContext)PredictionContext.EMPTY;
386         ATNConfigSet configs = new OrderedATNConfigSet();
387         for (int i=0; i<p.getNumberOfTransitions(); i++) {
388             ATNState target = p.transition(i).target;
389             LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
390             closure(input, c, configs, false, false, false);
391         }
392         return configs;
393     }
394 
395     /**
396      * @uml
397      * Since the alternatives within any lexer decision are ordered by
398      * preference, this method stops pursuing the closure as soon as an accept
399      * state is reached. After the first accept state is reached by depth-first
400      * search from {@code config}, all other (potentially reachable) states for
401      * this rule would have a lower priority.
402      *
403      *  @return {@code true} if an accept state is reached, otherwise
404      * {@code false}.
405      */
406     protected bool closure(CharStream input, LexerATNConfig config, ATNConfigSet configs,
407                            bool currentAltReachedAcceptState, bool speculative, bool treatEofAsEpsilon)
408     {
409         debug(LexerATNSimulator)
410             writefln("closure(%s)", config);
411         if (cast(RuleStopState)config.state) {
412             debug(LexerATNSimulator)  {
413                 if (recog !is null) {
414                     writefln("closure at %1$s rule stop %2$s\n", recog.getRuleNames()[config.state.ruleIndex], config);
415                 }
416                 else {
417                     writefln("closure at rule stop %s\n", config);
418                 }
419             }
420 
421             if (config.context is null || config.context.hasEmptyPath()) {
422                 if (config.context is null || config.context.isEmpty()) {
423                     configs.add(config);
424                     return true;
425                 }
426                 else {
427                     configs.add(new LexerATNConfig(config,
428                                                    config.state,
429                                                    cast(PredictionContext)PredictionContext.EMPTY));
430                     currentAltReachedAcceptState = true;
431                 }
432             }
433             if (config.context !is null && !config.context.isEmpty() ) {
434                 for (auto i = 0; i < config.context.size; i++) {
435                     if (config.context.getReturnState(i) != PredictionContext.EMPTY_RETURN_STATE) {
436                         PredictionContext newContext = config.context.getParent(i); // "pop" return state
437                         ATNState returnState = atn.states[config.context.getReturnState(i)];
438                         LexerATNConfig c = new LexerATNConfig(config, returnState, newContext);
439                         currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState,
440                                                                speculative, treatEofAsEpsilon);
441                     }
442                 }
443             }
444 
445             return currentAltReachedAcceptState;
446         }
447 
448         // optimization
449         if (!config.state.onlyHasEpsilonTransitions) {
450             if (!currentAltReachedAcceptState || !config.hasPassedThroughNonGreedyDecision()) {
451                 configs.add(config);
452             }
453         }
454 
455         ATNState p = config.state;
456         for (int i=0; i<p.getNumberOfTransitions(); i++) {
457             Transition t = p.transition(i);
458             LexerATNConfig c = getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon);
459             if (c !is null) {
460                 currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState,
461                                                        speculative, treatEofAsEpsilon);
462             }
463         }
464         return currentAltReachedAcceptState;
465     }
466 
467     /**
468      * side-effect: can alter configs.hasSemanticContext
469      */
470     protected LexerATNConfig getEpsilonTarget(CharStream input, LexerATNConfig config, Transition t,
471                                               ref ATNConfigSet configs, bool speculative, bool treatEofAsEpsilon)
472     {
473         LexerATNConfig c = null;
474         switch (t.getSerializationType) {
475         case TransitionStates.RULE:
476             RuleTransition ruleTransition = cast(RuleTransition)t;
477             PredictionContext newContext =
478                 SingletonPredictionContext.create(config.context, ruleTransition.followState.stateNumber);
479             c = new LexerATNConfig(config, t.target, newContext);
480             break;
481         case TransitionStates.PRECEDENCE:
482             throw new UnsupportedOperationException("Precedence predicates are not supported in lexers.");
483         case TransitionStates.PREDICATE:
484             /*  Track traversing semantic predicates. If we traverse,
485                 we cannot add a DFA state for this "reach" computation
486                 because the DFA would not test the predicate again in the
487                 future. Rather than creating collections of semantic predicates
488                 like v3 and testing them on prediction, v4 will test them on the
489                 fly all the time using the ATN not the DFA. This is slower but
490                 semantically it's not used that often. One of the key elements to
491                 this predicate mechanism is not adding DFA states that see
492                 predicates immediately afterwards in the ATN. For example,
493 
494                 a : ID {p1}? | ID {p2}? ;
495 
496                 should create the start state for rule 'a' (to save start state
497                 competition), but should not create target of ID state. The
498                 collection of ATN states the following ID references includes
499                 states reached by traversing predicates. Since this is when we
500                 test them, we cannot cash the DFA state target of ID.
501             */
502             PredicateTransition pt = cast(PredicateTransition)t;
503             debug(LexerATNSimulator) {
504                 writefln("EVAL rule %1$s:%2$s", pt.ruleIndex, pt.predIndex);
505             }
506             configs.hasSemanticContext = true;
507             if (evaluatePredicate(input, pt.ruleIndex, pt.predIndex, speculative)) {
508                 c = new LexerATNConfig(config, t.target);
509             }
510             break;
511 
512         case TransitionStates.ACTION:
513             if (config.context is null || config.context.hasEmptyPath()) {
514                 // execute actions anywhere in the start rule for a token.
515                 //
516                 // TODO: if the entry rule is invoked recursively, some
517                 // actions may be executed during the recursive call. The
518                 // problem can appear when hasEmptyPath() is true but
519                 // isEmpty() is false. In this case, the config needs to be
520                 // split into two contexts - one with just the empty path
521                 // and another with everything but the empty path.
522                 // Unfortunately, the current algorithm does not allow
523                 // getEpsilonTarget to return two configurations, so
524                 // additional modifications are needed before we can support
525                 // the split operation.
526                 LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append(config.getLexerActionExecutor(), atn.lexerActions[(cast(ActionTransition)t).actionIndex]);
527                 c = new LexerATNConfig(config, t.target, lexerActionExecutor);
528                 break;
529             }
530             else {
531                 // ignore actions in referenced rules
532                 c = new LexerATNConfig(config, t.target);
533                 break;
534             }
535 
536         case TransitionStates.EPSILON:
537             c = new LexerATNConfig(config, t.target);
538             break;
539 
540         case TransitionStates.ATOM:
541         case TransitionStates.RANGE:
542         case TransitionStates.SET:
543             if (treatEofAsEpsilon) {
544                 if (t.matches(IntStreamConstant.EOF, 0, 0xfffe)) {
545                     c = new LexerATNConfig(config, t.target);
546                     break;
547                 }
548             }
549 
550             break;
551         default: {}
552         }
553 
554         return c;
555     }
556 
557     /**
558      * Evaluate a predicate specified in the lexer.
559      *
560      * If {@code speculative} is {@code true}, this method was called before
561      * {@link #consume} for the matched character. This method should call
562      * {@link #consume} before evaluating the predicate to ensure position
563      * sensitive values, including {@link Lexer#getText}, {@link Lexer#getLine},
564      * and {@link Lexer#getCharPositionInLine}, properly reflect the current
565      * lexer state. This method should restore {@code input} and the simulator
566      * to the original state before returning (i.e. undo the actions made by the
567      * call to {@link #consume}).
568      *
569      *  @param input The input stream.
570      *  @param ruleIndex The rule containing the predicate.
571      *  @param predIndex The index of the predicate within the rule.
572      *  @param speculative {@code true} if the current index in {@code input} is
573      *  one character before the predicate's location.
574      *
575      *  @return {@code true} if the specified predicate evaluates to
576      * {@code true}.
577      */
578     protected bool evaluatePredicate(CharStream input, int ruleIndex, int predIndex, bool speculative)
579     {
580         // assume true if no recognizer was provided
581         if (recog is null) {
582             return true;
583         }
584 
585         if (!speculative) {
586             return recog.sempred(null, ruleIndex, predIndex);
587         }
588 
589         int savedCharPositionInLine = charPositionInLine;
590         int savedLine = line;
591         auto index = input.index();
592         int marker = input.mark();
593         try {
594             consume(input);
595             return recog.sempred(null, ruleIndex, predIndex);
596         }
597         finally {
598             charPositionInLine = savedCharPositionInLine;
599             line = savedLine;
600             input.seek(index);
601             input.release(marker);
602         }
603     }
604 
605     public void captureSimState(ref SimState settings, CharStream input, DFAState dfaState)
606     {
607         settings.index = to!int(input.index);
608         settings.line = line;
609         settings.charPos = charPositionInLine;
610         settings.dfaState = dfaState;
611     }
612 
613     protected DFAState addDFAEdge(DFAState from, int t, ATNConfigSet q)
614     {
615         /* leading to this call, ATNConfigSet.hasSemanticContext is used as a
616          * marker indicating dynamic predicate evaluation makes this edge
617          * dependent on the specific input sequence, so the static edge in the
618          * DFA should be omitted. The target DFAState is still created since
619          * execATN has the ability to resynchronize with the DFA state cache
620          * following the predicate evaluation step.
621          *
622          * TJP notes: next time through the DFA, we see a pred again and eval.
623          * If that gets us to a previously created (but dangling) DFA
624          * state, we can continue in pure DFA mode from there.
625          */
626         bool suppressEdge = q.hasSemanticContext;
627         q.hasSemanticContext = false;
628 
629         DFAState to = addDFAState(q);
630 
631         if (suppressEdge) {
632             return to;
633         }
634 
635         addDFAEdge(from, t, to);
636         return to;
637     }
638 
639     protected void addDFAEdge(DFAState p, int t, DFAState q)
640     {
641         if (t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) {
642             // Only track edges within the DFA bounds
643             return;
644         }
645 
646         debug(LexerATNSimulator) {
647             writefln("EDGE %1$s -> %2$s upon %3$s", p, q, cast(dchar)t);
648         }
649 
650         synchronized (p) {
651             if (p.edges is null) {
652                 //  make room for tokens 1..n and -1 masquerading as index 0
653                 p.edges = new DFAState[MAX_DFA_EDGE-MIN_DFA_EDGE+1];
654             }
655             p.edges[t - MIN_DFA_EDGE] = q; // connect
656         }
657     }
658 
659     /**
660      * Add a new DFA state if there isn't one with this set of
661      * configurations already. This method also detects the first
662      * configuration containing an ATN rule stop state. Later, when
663      * traversing the DFA, we will know which rule to accept.
664      */
665     protected DFAState addDFAState(ATNConfigSet configs)
666     {
667         /* the lexer evaluates predicates on-the-fly; by this point configs
668          * should not contain any configurations with unevaluated predicates.
669          */
670         assert(!configs.hasSemanticContext);
671         DFAState proposed = new DFAState(configs);
672         ATNConfig firstConfigWithRuleStopState;
673         foreach (ATNConfig c; configs.configs) {
674             if (cast(RuleStopState)c.state) {
675                 firstConfigWithRuleStopState = c;
676                 break;
677             }
678         }
679         if (firstConfigWithRuleStopState) {
680             proposed.isAcceptState = true;
681             proposed.lexerActionExecutor = (cast(LexerATNConfig)firstConfigWithRuleStopState).getLexerActionExecutor();
682             proposed.prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex];
683         }
684         DFA dfa = decisionToDFA[mode];
685         //DFAState existing = dfa.states[proposed];
686         if (proposed in dfa.states)
687             return dfa.states[proposed];
688         DFAState newState = proposed;
689         newState.stateNumber = to!int(dfa.states.length);
690         configs.readonly(true);
691         newState.configs = configs;
692         dfa.states[newState] =  newState;
693         return newState;
694     }
695 
696     public DFA getDFA(int mode)
697     {
698         return decisionToDFA[mode];
699     }
700 
701     public string getText(CharStream input)
702     {
703         // index is first lookahead char, don't include.
704         return input.getText(Interval.of(startIndex, to!int(input.index) - 1));
705     }
706 
707     public int getLine()
708     {
709         return line;
710     }
711 
712     public void setLine(int line)
713     {
714         this.line = line;
715     }
716 
717     public int getCharPositionInLine()
718     {
719         return charPositionInLine;
720     }
721 
722     public void setCharPositionInLine(int charPositionInLine)
723     {
724         this.charPositionInLine = charPositionInLine;
725     }
726 
727     public void consume(CharStream input)
728     {
729         int curChar = input.LA(1);
730         if (curChar == '\n') {
731             line++;
732             charPositionInLine=0;
733         } else {
734             charPositionInLine++;
735         }
736         input.consume();
737     }
738 
739     public string getTokenName(int t)
740     {
741         if (t == -1) return "EOF";
742         //if ( atn.g!=null ) return atn.g.getTokenDisplayName(t);
743         return format("'%s'", cast(dchar)t);
744     }
745 
746 }