1 /*
2  * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.atn.LexerATNSimulator;
8 
9 import std.conv;
10 import std.format;
11 import std.stdio;
12 import antlr.v4.runtime.atn.ATNSimulator;
13 import antlr.v4.runtime.IntStreamConstant;
14 import antlr.v4.runtime.Lexer;
15 import antlr.v4.runtime.UnsupportedOperationException;
16 import antlr.v4.runtime.dfa.DFA;
17 import antlr.v4.runtime.dfa.DFAState;
18 import antlr.v4.runtime.atn.ATN;
19 import antlr.v4.runtime.atn.ATNState;
20 import antlr.v4.runtime.atn.ATNConfigSet;
21 import antlr.v4.runtime.atn.ATNConfig;
22 import antlr.v4.runtime.atn.LexerActionExecutor;
23 import antlr.v4.runtime.atn.LexerATNConfig;
24 import antlr.v4.runtime.atn.PredictionContext;
25 import antlr.v4.runtime.atn.RuleTransition;
26 import antlr.v4.runtime.atn.RuleStopState;
27 import antlr.v4.runtime.atn.PredictionContextCache;
28 import antlr.v4.runtime.atn.SimState;
29 import antlr.v4.runtime.atn.SingletonPredictionContext;
30 import antlr.v4.runtime.atn.PredicateTransition;
31 import antlr.v4.runtime.atn.Transition;
32 import antlr.v4.runtime.atn.TransitionStates;
33 import antlr.v4.runtime.atn.ActionTransition;
34 import antlr.v4.runtime.atn.OrderedATNConfigSet;
35 import antlr.v4.runtime.CharStream;
36 import antlr.v4.runtime.LexerNoViableAltException;
37 import antlr.v4.runtime.Token;
38 import antlr.v4.runtime.TokenConstantDefinition;
39 import antlr.v4.runtime.misc;
40 
41 /**
42  * "dup" of ParserInterpreter
43  */
44 class LexerATNSimulator : ATNSimulator
45 {
46 
47     public static immutable int MIN_DFA_EDGE = 0;
48 
49     public static immutable int MAX_DFA_EDGE = 127;
50 
51     protected Lexer recog;
52 
53     /**
54      * @uml
55      * The current token's starting index into the character stream.
56      * Shared across DFA to ATN simulation in case the ATN fails and the
57      * DFA did not have a previous accept state. In this case, we use the
58      * ATN-generated exception object.
59      */
60     protected int startIndex = -1;
61 
62     /**
63      * @uml
64      * line number 1..n within the input
65      */
66     protected int line = 1;
67 
68     /**
69      * @uml
70      * The index of the character relative to the beginning of the line 0..n-1
71      */
72     protected int charPositionInLine = 0;
73 
74     public DFA[] decisionToDFA;
75 
76     protected int mode = Lexer.DEFAULT_MODE;
77 
78     protected SimState prevAccept;
79 
80     public static int match_calls = 0;
81 
82     public this(ATN atn, DFA[] decisionToDFA, PredictionContextCache sharedContextCache)
83     {
84         this(null, atn, decisionToDFA,sharedContextCache);
85     }
86 
87     public this(Lexer recog, ATN atn, DFA[] decisionToDFA, PredictionContextCache sharedContextCache)
88     {
89         super(atn, sharedContextCache);
90         this.decisionToDFA = decisionToDFA;
91         this.recog = recog;
92     }
93 
94     public void copyState(LexerATNSimulator simulator)
95     {
96         this.charPositionInLine = simulator.charPositionInLine;
97         this.line = simulator.line;
98         this.mode = simulator.mode;
99         this.startIndex = simulator.startIndex;
100     }
101 
102     public int match(CharStream input, int mode)
103     {
104         match_calls++;
105         this.mode = mode;
106         int mark = input.mark;
107         try {
108             this.startIndex = input.index;
109             this.prevAccept.reset;
110             DFA dfa = decisionToDFA[mode];
111             if (dfa.s0 is null) {
112                 return matchATN(input);
113             }
114             else {
115                 return execATN(input, dfa.s0);
116             }
117         }
118         finally {
119             input.release(mark);
120         }
121     }
122 
123     /**
124      * @uml
125      * @override
126      */
127     public override void reset()
128     {
129         prevAccept.reset();
130         startIndex = -1;
131         line = 1;
132         charPositionInLine = 0;
133         mode = Lexer.DEFAULT_MODE;
134     }
135 
136     /**
137      * @uml
138      * @override
139      */
140     public override void clearDFA()
141     {
142         for (int d = 0; d < decisionToDFA.length; d++) {
143             decisionToDFA[d] = new DFA(atn.getDecisionState(d), d);
144         }
145     }
146 
147     protected int matchATN(CharStream input)
148     {
149         ATNState startState = atn.modeToStartState[mode];
150         int old_mode = mode;
151         ATNConfigSet s0_closure = computeStartState(input, startState);
152         bool suppressEdge = s0_closure.hasSemanticContext;
153         s0_closure.hasSemanticContext = false;
154         DFAState next = addDFAState(s0_closure);
155         if (!suppressEdge) {
156             decisionToDFA[mode].s0 = next;
157         }
158         int predict = execATN(input, next);
159 
160         debug(LexerATNSimulator)
161             writefln("DFA after matchATN:\n%1$s\n", decisionToDFA[old_mode].toLexerString());
162         return predict;
163     }
164 
165     protected int execATN(CharStream input, DFAState ds0)
166     {
167         debug(LexerATNSimulator) {
168             writefln("enter exec index %s from %s", input.index, ds0.configs);
169         }
170 
171         if (ds0.isAcceptState) {
172             // allow zero-length tokens
173             captureSimState(prevAccept, input, ds0);
174         }
175         int t = input.LA(1);
176         DFAState s = ds0; // s is current/from DFA state
177 
178         while (true) { // while more work
179             debug(LexerATNSimulator) {
180                 writefln("execATN loop starting closure: %s\n", s.configs);
181             }
182             // As we move src->trg, src->trg, we keep track of the previous trg to
183             // avoid looking up the DFA state again, which is expensive.
184             // If the previous target was already part of the DFA, we might
185             // be able to avoid doing a reach operation upon t. If s!=null,
186             // it means that semantic predicates didn't prevent us from
187             // creating a DFA state. Once we know s!=null, we check to see if
188             // the DFA state has an edge already for t. If so, we can just reuse
189             // it's configuration set; there's no point in re-computing it.
190             // This is kind of like doing DFA simulation within the ATN
191             // simulation because DFA simulation is really just a way to avoid
192             // computing reach/closure sets. Technically, once we know that
193             // we have a previously added DFA state, we could jump over to
194             // the DFA simulator. But, that would mean popping back and forth
195             // a lot and making things more complicated algorithmically.
196             // This optimization makes a lot of sense for loops within DFA.
197             // A character will take us back to an existing DFA state
198             // that already has lots of edges out of it. e.g., .* in comments.
199             DFAState target = getExistingTargetState(s, t);
200             if (target is null) {
201                 target = computeTargetState(input, s, t);
202             }
203             if (target == ERROR) {
204                 break;
205             }
206             // If this is a consumable input element, make sure to consume before
207             // capturing the accept state so the input index, line, and char
208             // position accurately reflect the state of the interpreter at the
209             // end of the token.
210             if (t != IntStreamConstant.EOF) {
211                 consume(input);
212             }
213 
214             if (target.isAcceptState) {
215                 captureSimState(prevAccept, input, target);
216                 if (t == IntStreamConstant.EOF) {
217                     break;
218                 }
219             }
220             t = input.LA(1);
221             s = target; // flip; current DFA target becomes new src/from state
222         }
223         return failOrAccept(prevAccept, input, s.configs, t);
224     }
225 
226     /**
227      * @uml
228      * Get an existing target state for an edge in the DFA. If the target state
229      * for the edge has not yet been computed or is otherwise not available,
230      * this method returns {@code null}.
231      *
232      *  @param s The current DFA state
233      *  @param t The next input symbol
234      *  @return The existing target DFA state for the given input symbol
235      * {@code t}, or {@code null} if the target state for this edge is not
236      * already cached
237      */
238     public DFAState getExistingTargetState(DFAState s, int t)
239     {
240 	if (s.edges is null || t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) {
241             return null;
242         }
243 
244         DFAState target = s.edges[t - MIN_DFA_EDGE];
245         if (target !is null)
246             debug(LexerATNSimulator) {
247                 writefln("reuse state %1$s"~
248                          " edge to %2$s", s.stateNumber, target.stateNumber);
249             }
250 
251         return target;
252     }
253 
254     /**
255      * @uml
256      * Compute a target state for an edge in the DFA, and attempt to add the
257      * computed state and corresponding edge to the DFA.
258      *
259      *  @param input The input stream
260      *  @param s The current DFA state
261      *  @param t The next input symbol
262      *
263      *  @return The computed target DFA state for the given input symbol
264      * {@code t}. If {@code t} does not lead to a valid DFA state, this method
265      * returns {@link #ERROR}.
266      */
267     protected DFAState computeTargetState(CharStream input, DFAState s, int t)
268     {
269 	ATNConfigSet reach = new OrderedATNConfigSet();
270 
271         // if we don't find an existing DFA state
272         // Fill reach starting from closure, following t transitions
273         getReachableConfigSet(input, s.configs, reach, t);
274 
275         if (reach.isEmpty()) { // we got nowhere on t from s
276             if (!reach.hasSemanticContext) {
277                 // we got nowhere on t, don't throw out this knowledge; it'd
278                 // cause a failover from DFA later.
279                 addDFAEdge(s, t, ERROR);
280             }
281 
282             // stop when we can't match any more char
283             return ERROR;
284         }
285 
286         // Add an edge from s to target DFA found/created for reach
287         return addDFAEdge(s, t, reach);
288     }
289 
290     protected int failOrAccept(SimState prevAccept, CharStream input, ATNConfigSet reach,
291                                int t)
292     {
293 	if (prevAccept.dfaState !is null) {
294             LexerActionExecutor lexerActionExecutor = prevAccept.dfaState.lexerActionExecutor;
295             accept(input, lexerActionExecutor, startIndex,
296                    prevAccept.index, prevAccept.line, prevAccept.charPos);
297             return prevAccept.dfaState.prediction;
298         }
299         else {
300             // if no accept and EOF is first char, return EOF
301             if (t==IntStreamConstant.EOF && input.index() == startIndex) {
302                 return TokenConstantDefinition.EOF;
303             }
304             throw new LexerNoViableAltException(recog, input, startIndex, reach);
305         }
306     }
307 
308     /**
309      * @uml
310      * Given a starting configuration set, figure out all ATN configurations
311      * we can reach upon input {@code t}. Parameter {@code reach} is a return
312      * parameter.
313      */
314     protected void getReachableConfigSet(CharStream input, ATNConfigSet closureATNConfigSet, ATNConfigSet reach,
315                                          int t)
316     {
317 	// this is used to skip processing for configs which have a lower priority
318         // than a config that already reached an accept state for the same rule
319         int skipAlt = ATN.INVALID_ALT_NUMBER;
320         foreach (ATNConfig c; closureATNConfigSet.configs) {
321             bool currentAltReachedAcceptState = c.alt == skipAlt;
322             if (currentAltReachedAcceptState && (cast(LexerATNConfig)c).hasPassedThroughNonGreedyDecision()) {
323                 continue;
324             }
325 
326             debug(LexerATNSimulator) {
327                 writefln("testing %s at %s\n", getTokenName(t), c.toString(recog, true));
328             }
329 
330             int n = c.state.getNumberOfTransitions();
331             for (int ti=0; ti<n; ti++) {               // for each transition
332                 Transition trans = c.state.transition(ti);
333                 ATNState target = getReachableTarget(trans, t);
334                 if (target !is null) {
335                     LexerActionExecutor lexerActionExecutor = (cast(LexerATNConfig)c).getLexerActionExecutor();
336                     if (lexerActionExecutor !is null) {
337                         lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex);
338                     }
339 
340                     bool treatEofAsEpsilon = t == IntStreamConstant.EOF;
341                     if (closure(input, new LexerATNConfig(cast(LexerATNConfig)c, target, lexerActionExecutor),
342                                 reach, currentAltReachedAcceptState, true, treatEofAsEpsilon)) {
343                         // any remaining configs for this alt have a lower priority than
344                         // the one that just reached an accept state.
345                         skipAlt = c.alt;
346                         break;
347                     }
348                 }
349             }
350         }
351 
352     }
353 
354     protected void accept(CharStream input, LexerActionExecutor lexerActionExecutor, int startIndex,
355                           int index, int line, int charPos)
356     {
357 	debug(LexerATNSimulator) {
358             writefln("ACTION %s\n", lexerActionExecutor);
359         }
360 
361         // seek to after last char in token
362         input.seek(index);
363         this.line = line;
364         this.charPositionInLine = charPos;
365 
366         if (lexerActionExecutor !is null && recog !is null) {
367             lexerActionExecutor.execute(recog, input, startIndex);
368         }
369     }
370 
371     protected ATNState getReachableTarget(Transition trans, int t)
372     {
373         if (trans.matches(t, 0, 0xfffe)) {
374             return trans.target;
375         }
376 
377         return null;
378     }
379 
380     protected ATNConfigSet computeStartState(CharStream input, ATNState p)
381     {
382         PredictionContext initialContext = cast(PredictionContext)PredictionContext.EMPTY;
383         ATNConfigSet configs = new OrderedATNConfigSet();
384         for (int i=0; i<p.getNumberOfTransitions(); i++) {
385             ATNState target = p.transition(i).target;
386             LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
387             closure(input, c, configs, false, false, false);
388         }
389         return configs;
390     }
391 
392     /**
393      * @uml
394      * Since the alternatives within any lexer decision are ordered by
395      * preference, this method stops pursuing the closure as soon as an accept
396      * state is reached. After the first accept state is reached by depth-first
397      * search from {@code config}, all other (potentially reachable) states for
398      * this rule would have a lower priority.
399      *
400      *  @return {@code true} if an accept state is reached, otherwise
401      * {@code false}.
402      */
403     protected bool closure(CharStream input, LexerATNConfig config, ATNConfigSet configs,
404                            bool currentAltReachedAcceptState, bool speculative, bool treatEofAsEpsilon)
405     {
406         if (cast(RuleStopState)config.state) {
407             debug(LexerATNSimulator)  {
408                 if (recog !is null) {
409                     writefln("closure at %1$s rule stop %2$s\n", recog.getRuleNames()[config.state.ruleIndex], config);
410                 }
411                 else {
412                     writefln("closure at rule stop %s\n", config);
413                 }
414             }
415 
416             if (config.context is null || config.context.hasEmptyPath()) {
417                 if (config.context is null || config.context.isEmpty()) {
418                     configs.add(config);
419                     return true;
420                 }
421                 else {
422                     configs.add(new LexerATNConfig(config,
423                                                    config.state,
424                                                    cast(PredictionContext)PredictionContext.EMPTY));
425                     currentAltReachedAcceptState = true;
426                 }
427             }
428             if (config.context !is null && !config.context.isEmpty() ) {
429                 for (auto i = 0; i < config.context.size; i++) {
430                     if (config.context.getReturnState(i) != PredictionContext.EMPTY_RETURN_STATE) {
431                         PredictionContext newContext = config.context.getParent(i); // "pop" return state
432                         ATNState returnState = atn.states[config.context.getReturnState(i)];
433                         LexerATNConfig c = new LexerATNConfig(config, returnState, newContext);
434                         currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState,
435                                                                speculative, treatEofAsEpsilon);
436                     }
437                 }
438             }
439 
440             return currentAltReachedAcceptState;
441         }
442 
443         // optimization
444         if (!config.state.onlyHasEpsilonTransitions) {
445             if (!currentAltReachedAcceptState || !config.hasPassedThroughNonGreedyDecision()) {
446                 configs.add(config);
447             }
448         }
449 
450         ATNState p = config.state;
451         for (int i=0; i<p.getNumberOfTransitions(); i++) {
452             Transition t = p.transition(i);
453             LexerATNConfig c = getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon);
454             if (c !is null) {
455                 currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState,
456                                                        speculative, treatEofAsEpsilon);
457             }
458         }
459         return currentAltReachedAcceptState;
460     }
461 
462     /**
463      * side-effect: can alter configs.hasSemanticContext
464      */
465     protected LexerATNConfig getEpsilonTarget(CharStream input, LexerATNConfig config, Transition t,
466         ref ATNConfigSet configs, bool speculative, bool treatEofAsEpsilon)
467     {
468         debug(LexerATNSimulator) {
469             import std.stdio;
470             writefln("LexerATNSimulator getEpsilonTarget t = %s, serType = %s", t, t.getSerializationType);
471         }
472 	LexerATNConfig c = null;
473         switch (t.getSerializationType) {
474         case TransitionStates.RULE:
475             RuleTransition ruleTransition = cast(RuleTransition)t;
476             PredictionContext newContext =
477                 SingletonPredictionContext.create(config.context, ruleTransition.followState.stateNumber);
478             c = new LexerATNConfig(config, t.target, newContext);
479             break;
480         case TransitionStates.PRECEDENCE:
481             throw new UnsupportedOperationException("Precedence predicates are not supported in lexers.");
482         case TransitionStates.PREDICATE:
483             /*  Track traversing semantic predicates. If we traverse,
484                 we cannot add a DFA state for this "reach" computation
485                 because the DFA would not test the predicate again in the
486                 future. Rather than creating collections of semantic predicates
487                 like v3 and testing them on prediction, v4 will test them on the
488                 fly all the time using the ATN not the DFA. This is slower but
489                 semantically it's not used that often. One of the key elements to
490                 this predicate mechanism is not adding DFA states that see
491                 predicates immediately afterwards in the ATN. For example,
492 
493                 a : ID {p1}? | ID {p2}? ;
494 
495                 should create the start state for rule 'a' (to save start state
496                 competition), but should not create target of ID state. The
497                 collection of ATN states the following ID references includes
498                 states reached by traversing predicates. Since this is when we
499                 test them, we cannot cash the DFA state target of ID.
500             */
501             PredicateTransition pt = cast(PredicateTransition)t;
502             debug(LexerATNSimulator) {
503                 writefln("EVAL rule %1$s:%2$s", pt.ruleIndex, pt.predIndex);
504             }
505             configs.hasSemanticContext = true;
506             if (evaluatePredicate(input, pt.ruleIndex, pt.predIndex, speculative)) {
507                 c = new LexerATNConfig(config, t.target);
508             }
509             break;
510 
511         case TransitionStates.ACTION:
512             if (config.context is null || config.context.hasEmptyPath()) {
513                 // execute actions anywhere in the start rule for a token.
514                 //
515                 // TODO: if the entry rule is invoked recursively, some
516                 // actions may be executed during the recursive call. The
517                 // problem can appear when hasEmptyPath() is true but
518                 // isEmpty() is false. In this case, the config needs to be
519                 // split into two contexts - one with just the empty path
520                 // and another with everything but the empty path.
521                 // Unfortunately, the current algorithm does not allow
522                 // getEpsilonTarget to return two configurations, so
523                 // additional modifications are needed before we can support
524                 // the split operation.
525                 LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append(config.getLexerActionExecutor(), atn.lexerActions[(cast(ActionTransition)t).actionIndex]);
526                 c = new LexerATNConfig(config, t.target, lexerActionExecutor);
527                 break;
528             }
529             else {
530                 // ignore actions in referenced rules
531                 c = new LexerATNConfig(config, t.target);
532                 break;
533             }
534 
535         case TransitionStates.EPSILON:
536             c = new LexerATNConfig(config, t.target);
537             break;
538 
539         case TransitionStates.ATOM:
540         case TransitionStates.RANGE:
541         case TransitionStates.SET:
542             if (treatEofAsEpsilon) {
543                 if (t.matches(IntStreamConstant.EOF, 0, 0xfffe)) {
544                     c = new LexerATNConfig(config, t.target);
545                     break;
546                 }
547             }
548 
549             break;
550         default: {}
551         }
552 
553         return c;
554     }
555 
556     /**
557      * Evaluate a predicate specified in the lexer.
558      *
559      * If {@code speculative} is {@code true}, this method was called before
560      * {@link #consume} for the matched character. This method should call
561      * {@link #consume} before evaluating the predicate to ensure position
562      * sensitive values, including {@link Lexer#getText}, {@link Lexer#getLine},
563      * and {@link Lexer#getCharPositionInLine}, properly reflect the current
564      * lexer state. This method should restore {@code input} and the simulator
565      * to the original state before returning (i.e. undo the actions made by the
566      * call to {@link #consume}).
567      *
568      *  @param input The input stream.
569      *  @param ruleIndex The rule containing the predicate.
570      *  @param predIndex The index of the predicate within the rule.
571      *  @param speculative {@code true} if the current index in {@code input} is
572      *  one character before the predicate's location.
573      *
574      *  @return {@code true} if the specified predicate evaluates to
575      * {@code true}.
576      */
577     protected bool evaluatePredicate(CharStream input, int ruleIndex, int predIndex, bool speculative)
578     {
579 	// assume true if no recognizer was provided
580         if (recog is null) {
581             return true;
582         }
583 
584         if (!speculative) {
585             return recog.sempred(null, ruleIndex, predIndex);
586         }
587 
588         int savedCharPositionInLine = charPositionInLine;
589         int savedLine = line;
590         int index = input.index();
591         int marker = input.mark();
592         try {
593             consume(input);
594             return recog.sempred(null, ruleIndex, predIndex);
595         }
596         finally {
597             charPositionInLine = savedCharPositionInLine;
598             line = savedLine;
599             input.seek(index);
600             input.release(marker);
601         }
602     }
603 
604     public void captureSimState(ref SimState settings, CharStream input, DFAState dfaState)
605     {
606         settings.index = input.index;
607         settings.line = line;
608         settings.charPos = charPositionInLine;
609         settings.dfaState = dfaState;
610     }
611 
612     protected DFAState addDFAEdge(DFAState from, int t, ATNConfigSet q)
613     {
614 	/* leading to this call, ATNConfigSet.hasSemanticContext is used as a
615          * marker indicating dynamic predicate evaluation makes this edge
616          * dependent on the specific input sequence, so the static edge in the
617          * DFA should be omitted. The target DFAState is still created since
618          * execATN has the ability to resynchronize with the DFA state cache
619          * following the predicate evaluation step.
620          *
621          * TJP notes: next time through the DFA, we see a pred again and eval.
622          * If that gets us to a previously created (but dangling) DFA
623          * state, we can continue in pure DFA mode from there.
624          */
625         bool suppressEdge = q.hasSemanticContext;
626         q.hasSemanticContext = false;
627 
628         DFAState to = addDFAState(q);
629 
630         if (suppressEdge) {
631             return to;
632         }
633 
634         addDFAEdge(from, t, to);
635         return to;
636     }
637 
638     protected void addDFAEdge(DFAState p, int t, DFAState q)
639     {
640 	if (t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) {
641             // Only track edges within the DFA bounds
642             return;
643         }
644 
645         debug(LexerATNSimulator) {
646             writefln("EDGE %1$s -> %2$s upon %3$s", p, q, t);
647         }
648 
649         synchronized (p) {
650             if (p.edges is null) {
651                 //  make room for tokens 1..n and -1 masquerading as index 0
652                 p.edges = new DFAState[MAX_DFA_EDGE-MIN_DFA_EDGE+1];
653             }
654             p.edges[t - MIN_DFA_EDGE] = q; // connect
655         }
656     }
657 
658     /**
659      * Add a new DFA state if there isn't one with this set of
660      * configurations already. This method also detects the first
661      * configuration containing an ATN rule stop state. Later, when
662      * traversing the DFA, we will know which rule to accept.
663      */
664     protected DFAState addDFAState(ATNConfigSet configs)
665     {
666         /* the lexer evaluates predicates on-the-fly; by this point configs
667          * should not contain any configurations with unevaluated predicates.
668          */
669         assert(!configs.hasSemanticContext);
670         DFAState proposed = new DFAState(configs);
671         ATNConfig firstConfigWithRuleStopState;
672         foreach (ATNConfig c; configs.configs) {
673             if (cast(RuleStopState)c.state)	{
674                 firstConfigWithRuleStopState = c;
675                 break;
676             }
677         }
678         if (firstConfigWithRuleStopState) {
679             proposed.isAcceptState = true;
680             proposed.lexerActionExecutor = (cast(LexerATNConfig)firstConfigWithRuleStopState).getLexerActionExecutor();
681             proposed.prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex];
682         }
683         DFA dfa = decisionToDFA[mode];
684         //DFAState existing = dfa.states[proposed];
685         if (proposed in dfa.states)
686             return dfa.states[proposed];
687         DFAState newState = proposed;
688         newState.stateNumber = to!int(dfa.states.length);
689         configs.readonly(true);
690         newState.configs = configs;
691         dfa.states[newState] =  newState;
692         return newState;
693     }
694 
695     public DFA getDFA(int mode)
696     {
697         return decisionToDFA[mode];
698     }
699 
700     public string getText(CharStream input)
701     {
702         // index is first lookahead char, don't include.
703         return input.getText(Interval.of(startIndex, input.index()-1));
704     }
705 
706     public int getLine()
707     {
708         return line;
709     }
710 
711     public void setLine(int line)
712     {
713         this.line = line;
714     }
715 
716     public int getCharPositionInLine()
717     {
718 	return charPositionInLine;
719     }
720 
721     public void setCharPositionInLine(int charPositionInLine)
722     {
723         this.charPositionInLine = charPositionInLine;
724     }
725 
726     public void consume(CharStream input)
727     {
728 	int curChar = input.LA(1);
729         if (curChar == '\n') {
730             line++;
731             charPositionInLine=0;
732         } else {
733             charPositionInLine++;
734         }
735         input.consume();
736     }
737 
738     public string getTokenName(int t)
739     {
740         if (t == -1) return "EOF";
741         //if ( atn.g!=null ) return atn.g.getTokenDisplayName(t);
742         return format("'%s'", t);
743     }
744 
745 }