antlr.v4.runtime.CommonTokenStream source code

1 /*
2  * Copyright (c) 2012-2018 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonTokenStream;
8 
9 import std.conv;
10 import std.stdio;
11 import antlr.v4.runtime.BufferedTokenStream;
12 import antlr.v4.runtime.Token;
13 import antlr.v4.runtime.TokenConstantDefinition;
14 import antlr.v4.runtime.TokenSource;
15 
16 /**
17  * This class extends {@link BufferedTokenStream} with functionality to filter
18  * token streams to tokens on a particular channel (tokens where
19  * {@link Token#getChannel} returns a particular value).
20  * <p>
21  * This token stream provides access to all tokens by index or when calling
22  * methods like {@link #getText}. The channel filtering is only used for code
23  * accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and
24  * {@link #LB}.</p>
25  *
26  * <p>
27  * By default, tokens are placed on the default channel
28  * ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the
29  * {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to
30  * call {@link Lexer#setChannel}.
31  * </p>
32  *
33  * <p>
34  * Note: lexer rules which use the {@code ->skip} lexer command or call
35  * {@link Lexer#skip} do not produce tokens at all, so input text matched by
36  * such a rule will not be available as part of the token stream, regardless of
37  * channel.</p>
38  */
39 class CommonTokenStream : BufferedTokenStream
40 {
41 
42     /**
43      * Specifies the channel to use for filtering tokens.
44      *
45      * <p>
46      * The default value is {@link Token#DEFAULT_CHANNEL}, which matches the
47      * default channel assigned to tokens created by the lexer.</p>
48      */
49     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
50 
51     /**
52      * Constructs a new {@link CommonTokenStream} using the specified token
53      * source and the default token channel ({@link Token#DEFAULT_CHANNEL}).
54      *
55      *  @param tokenSource The token source.
56      */
57     public this(TokenSource tokenSource)
58     {
59         super(tokenSource);
60     }
61 
62     /**
63      * Constructs a new {@link CommonTokenStream} using the specified token
64      * source and filtering tokens to the specified channel. Only tokens whose
65      * {@link Token#getChannel} matches {@code channel} or have the
66      * {@link Token#getType} equal to {@link Token#EOF} will be returned by the
67      * token stream lookahead methods.
68      *
69      * @param tokenSource The token source.
70      * @param channel The channel to use for filtering tokens.
71      */
72     public this(TokenSource tokenSource, int channel)
73     {
74         this(tokenSource);
75         this.channel = channel;
76     }
77 
78     /**
79      * @uml
80      * @override
81      */
82     protected override size_t adjustSeekIndex(size_t i)
83     {
84         return nextTokenOnChannel(i, channel);
85     }
86 
87     /**
88      * @uml
89      * @override
90      */
91     protected override Token LB(int k)
92     {
93         if (k == 0 || (index - k) < 0 ) return null;
94 
95         auto i = index;
96         int n = 1;
97         // find k good tokens looking backwards
98         while (n <= k && i > 0) {
99             // skip off-channel tokens
100             i = previousTokenOnChannel(i - 1, channel);
101             n++;
102         }
103         if (i < 0) return null;
104         return tokens[i];
105     }
106 
107     /**
108      * @uml
109      * @override
110      */
111     public override Token LT(int k)
112     {
113         debug
114             writefln("enter LT(%s) on channel = %s, p = %s", k, channel, index);
115         lazyInit;
116         if (k == 0 ) return null;
117         if (k < 0) return LB(-k);
118         auto i = index;
119         int n = 1; // we know tokens[p] is a good one
120         // find k good tokens
121         while (n < k) {
122             // skip off-channel tokens, but make sure to not look past EOF
123             if (sync(i + 1)) {
124                 i = nextTokenOnChannel(i + 1, channel);
125             }
126             n++;
127         }
128         //   if ( i>range ) range = i;
129         debug
130             writefln("enter end LT(%s): %s", i, tokens[i]);
131         return tokens[i];
132     }
133 
134     /**
135      * Count EOF just once.
136      */
137     public int getNumberOfOnChannelTokens()
138     {
139         int n = 0;
140         fill;
141         foreach (t; tokens) {
142             if (t.getChannel == channel)
143                 n++;
144             if (t.getType == TokenConstantDefinition.EOF)
145                 break;
146         }
147         return n;
148     }
149 
150 }