antlr.v4.runtime.CommonTokenStream source code

1 /*
2  * Copyright (c) 2012-2018 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonTokenStream;
8 
9 import std.conv;
10 import std.stdio;
11 import antlr.v4.runtime.BufferedTokenStream;
12 import antlr.v4.runtime.Token;
13 import antlr.v4.runtime.TokenConstantDefinition;
14 import antlr.v4.runtime.TokenSource;
15 
16 // Class CommonTokenStream
17 /**
18  * This class extends {@link BufferedTokenStream} with functionality to filter
19  * token streams to tokens on a particular channel (tokens where
20  * {@link Token#getChannel} returns a particular value).
21  * <p>
22  * This token stream provides access to all tokens by index or when calling
23  * methods like {@link #getText}. The channel filtering is only used for code
24  * accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and
25  * {@link #LB}.</p>
26  *
27  * <p>
28  * By default, tokens are placed on the default channel
29  * ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the
30  * {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to
31  * call {@link Lexer#setChannel}.
32  * </p>
33  *
34  * <p>
35  * Note: lexer rules which use the {@code ->skip} lexer command or call
36  * {@link Lexer#skip} do not produce tokens at all, so input text matched by
37  * such a rule will not be available as part of the token stream, regardless of
38  * channel.</p>
39  */
40 class CommonTokenStream : BufferedTokenStream
41 {
42 
43     /**
44      * Specifies the channel to use for filtering tokens.
45      *
46      * <p>
47      * The default value is {@link Token#DEFAULT_CHANNEL}, which matches the
48      * default channel assigned to tokens created by the lexer.</p>
49      */
50     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
51 
52     /**
53      * Constructs a new {@link CommonTokenStream} using the specified token
54      * source and the default token channel ({@link Token#DEFAULT_CHANNEL}).
55      *
56      *  @param tokenSource The token source.
57      */
58     public this(TokenSource tokenSource)
59     {
60         super(tokenSource);
61     }
62 
63     /**
64      * Constructs a new {@link CommonTokenStream} using the specified token
65      * source and filtering tokens to the specified channel. Only tokens whose
66      * {@link Token#getChannel} matches {@code channel} or have the
67      * {@link Token#getType} equal to {@link Token#EOF} will be returned by the
68      * token stream lookahead methods.
69      *
70      * @param tokenSource The token source.
71      * @param channel The channel to use for filtering tokens.
72      */
73     public this(TokenSource tokenSource, int channel)
74     {
75         this(tokenSource);
76         this.channel = channel;
77     }
78 
79     /**
80      * @uml
81      * @override
82      */
83     protected override int adjustSeekIndex(int i)
84     {
85         return nextTokenOnChannel(i, channel);
86     }
87 
88     /**
89      * @uml
90      * @override
91      */
92     protected override Token LB(int k)
93     {
94         if (k == 0 || (index - k) < 0 ) return null;
95 
96         int i = index;
97         int n = 1;
98         // find k good tokens looking backwards
99         while (n <= k && i > 0) {
100             // skip off-channel tokens
101             i = previousTokenOnChannel(i - 1, channel);
102             n++;
103         }
104         if (i < 0) return null;
105         return tokens[i];
106     }
107 
108     /**
109      * @uml
110      * @override
111      */
112     public override Token LT(int k)
113     {
114         debug
115             writefln("enter LT(%s) on channel = %s, p = %s", k, channel, index);
116         lazyInit();
117         if (k == 0 ) return null;
118         if (k < 0) return LB(-k);
119         int i = index;
120         int n = 1; // we know tokens[p] is a good one
121         // find k good tokens
122         while (n < k) {
123             // skip off-channel tokens, but make sure to not look past EOF
124             if (sync(i + 1)) {
125                 i = nextTokenOnChannel(i + 1, channel);
126             }
127             n++;
128         }
129         //   if ( i>range ) range = i;
130         debug
131             writefln("enter end LT(%s): %s", i, tokens[i]);
132         return tokens[i];
133     }
134 
135     /**
136      * Count EOF just once.
137      */
138     public int getNumberOfOnChannelTokens()
139     {
140         int n = 0;
141         fill();
142         foreach (t; tokens) {
143             if (t.getChannel == channel)
144                 n++;
145             if (t.getType == TokenConstantDefinition.EOF)
146                 break;
147         }
148         return n;
149     }
150 
151 }