1 /*
2  * Copyright (c) 2012-2020 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonToken;
8 
9 import antlr.v4.runtime.CharStream;
10 import antlr.v4.runtime.Token;
11 import antlr.v4.runtime.TokenConstantDefinition;
12 import antlr.v4.runtime.TokenSource;
13 import antlr.v4.runtime.WritableToken;
14 import antlr.v4.runtime.misc.Interval;
15 import std.array;
16 import std.container : DList;
17 import std.conv;
18 import std.typecons;
19 import std.variant;
20 
21 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
22 
23 /**
24  * TODO add class description
25  */
26 class CommonToken : WritableToken
27 {
28 
29     /**
30      * An empty {@link Pair} which is used as the default value of
31      * {@link #source} for tokens that do not have a source.
32      */
33     protected static TokenFactorySourcePair EMPTY_SOURCE;
34 
35     /**
36      * This is the backing field for {@link #getType} and {@link #setType}.
37      */
38     protected int type;
39 
40     /**
41      * This is the backing field for {@link #getLine} and {@link #setLine}.
42      */
43     protected int line;
44 
45     /**
46      * This is the backing field for {@link #getCharPositionInLine} and
47      * {@link #setCharPositionInLine}.
48      */
49     protected int charPositionInLine = -1;
50 
51     /**
52      * This is the backing field for {@link #getChannel} and
53      * {@link #setChannel}.
54      */
55     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
56 
57     /**
58      * This is the backing field for {@link #getTokenSource} and
59      * {@link #getInputStream}.
60      *
61      * <p>
62      * These properties share a field to reduce the memory footprint of
63      * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
64      * the same source and input stream share a reference to the same
65      * {@link Pair} containing these values.</p>
66      */
67     protected TokenFactorySourcePair source;
68 
69     /**
70      * This is the backing field for {@link #getText} when the token text is
71      * explicitly set in the constructor or via {@link #setText}.
72      *
73      *  @see #getText()
74      */
75     protected Variant text;
76 
77     /**
78      * This is the backing field for {@link #getTokenIndex} and
79      * {@link #setTokenIndex}.
80      */
81     protected size_t index = size_t.max;
82 
83     /**
84      * This is the backing field for {@link #getStartIndex} and
85      * {@link #setStartIndex}.
86      * @uml
87      * @read
88      * @write
89      */
90     protected size_t startIndex_;
91 
92     /**
93      * This is the backing field for {@link #getStopIndex} and
94      * {@link #setStopIndex}.
95      * @uml
96      * @read
97      * @write
98      */
99     protected size_t stopIndex_;
100 
101     /**
102      * Constructs a new {@link CommonToken} with the specified token type.
103      *
104      *  @param type The token type.
105      */
106     public this(int type)
107     {
108         this.type = type;
109         this.source = EMPTY_SOURCE;
110     }
111 
112     public this(TokenFactorySourcePair source, int type, int channel, size_t start, size_t stop)
113     {
114         this.source = source;
115         this.type = type;
116         this.channel = channel;
117         this.startIndex_ = start;
118         this.stopIndex_ = stop;
119         if (source.a)
120         {
121             this.line = source.a.getLine;
122             this.charPositionInLine = source.a.getCharPositionInLine;
123         }
124     }
125 
126     /**
127      * Constructs a new {@link CommonToken} with the specified token type and
128      * text.
129      *
130      *  @param type The token type.
131      *  @param text The text of the token.
132      */
133     public this(int type, Variant text)
134     {
135         this.type = type;
136         this.channel = TokenConstantDefinition.DEFAULT_CHANNEL;
137         this.text = text;
138         this.source = EMPTY_SOURCE;
139     }
140 
141     /**
142      * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
143      *   *
144      * <p>
145      * If {@code oldToken} is also a {@link CommonToken} instance, the newly
146      * constructed token will share a reference to the {@link #text} field and
147      * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
148      * be assigned the result of calling {@link #getText}, and {@link #source}
149      * will be constructed from the result of {@link Token#getTokenSource} and
150      * {@link Token#getInputStream}.</p>
151      *
152      *  @param oldToken The token to copy.
153      */
154     public this(Token oldToken)
155     {
156         type = oldToken.getType;
157         line = oldToken.getLine;
158         index = oldToken.getTokenIndex;
159         charPositionInLine = oldToken.getCharPositionInLine;
160         channel = oldToken.getChannel;
161         startIndex_ = oldToken.startIndex;
162         stopIndex_ = oldToken.stopIndex;
163 
164         if (cast(CommonToken) oldToken)
165         {
166             text = (cast(CommonToken) oldToken).text;
167             source = (cast(CommonToken) oldToken).source;
168         }
169         else
170         {
171             text = oldToken.getText;
172             TokenFactorySourcePair sourceNew = tuple(oldToken.getTokenSource,
173                     oldToken.getInputStream);
174             source = sourceNew;
175         }
176     }
177 
178     public int getType()
179     {
180         return type;
181     }
182 
183     public void setLine(int line)
184     {
185         this.line = line;
186     }
187 
188     /**
189      * @uml
190      * @override
191      */
192     public override Variant getText()
193     {
194         Variant Null;
195         if (text !is Null)
196         {
197             return text;
198         }
199 
200         CharStream input = getInputStream;
201         if (input is null)
202             return Null;
203         auto n = input.size;
204         if (startIndex_ < n && stopIndex_ < n)
205         {
206             Variant v = input.getText(Interval.of(to!int(startIndex_), to!int(stopIndex_)));
207             return v;
208         }
209         else
210         {
211             Variant v = "<EOF>";
212             return v;
213         }
214     }
215 
216     /**
217      * Explicitly set the text for this token. If {code text} is not
218      * {@code null}, then {@link #getText} will return this value rather than
219      * extracting the text from the input.
220      *
221      *  @param text The explicit text of the token, or {@code null} if the text
222      * should be obtained from the input along with the start and stop indexes
223      * of the token.
224      * @uml
225      * @override
226      */
227     public override void setText(Variant text)
228     {
229         this.text = text;
230     }
231 
232     /**
233      * @uml
234      * @override
235      */
236     public override int getLine()
237     {
238         return line;
239     }
240 
241     /**
242      * @uml
243      * @override
244      */
245     public override int getCharPositionInLine()
246     {
247         return charPositionInLine;
248     }
249 
250     public void setCharPositionInLine(int charPositionInLine)
251     {
252         this.charPositionInLine = charPositionInLine;
253     }
254 
255     /**
256      * @uml
257      * @override
258      */
259     public override int getChannel()
260     {
261         return channel;
262     }
263 
264     /**
265      * @uml
266      * @override
267      */
268     public override void setChannel(int channel)
269     {
270         this.channel = channel;
271     }
272 
273     /**
274      * @uml
275      * @override
276      */
277     public override void setType(int type)
278     {
279         this.type = type;
280     }
281 
282     /**
283      * @uml
284      * @override
285      */
286     public override size_t getTokenIndex()
287     {
288         return index;
289     }
290 
291     /**
292      * @uml
293      * @override
294      */
295     public override void setTokenIndex(size_t index)
296     {
297         this.index = index;
298     }
299 
300     /**
301      * @uml
302      * @override
303      */
304     public override TokenSource getTokenSource()
305     {
306         return source.a;
307     }
308 
309     /**
310      * @uml
311      * @override
312      */
313     public override CharStream getInputStream()
314     {
315         return source.b;
316     }
317 
318     /**
319      * @uml
320      * @override
321      */
322     public override string toString()
323     {
324         import std.format : format;
325 
326         string channelStr = "";
327         if (channel > 0)
328         {
329             channelStr = ",channel=" ~ to!string(channel);
330         }
331         auto txt = getText.get!(string);
332         if (txt.length > 0)
333         {
334             txt = txt.replace("\n", "\\n");
335             txt = txt.replace("\r", "\\r");
336             txt = txt.replace("\t", "\\t");
337         }
338         else
339         {
340             txt = "<no text>";
341         }
342         return format!"[@%s,%s:%s='%s',<%s>%s,%s:%s]"(cast(int)getTokenIndex,
343             cast(int)startIndex_,
344             cast(int)stopIndex_,
345             txt, type, channelStr,
346             line, getCharPositionInLine
347             );
348     }
349 
350     public final size_t startIndex()
351     {
352         return this.startIndex_;
353     }
354 
355     public final void startIndex(size_t startIndex)
356     {
357         this.startIndex_ = startIndex;
358     }
359 
360     public final size_t stopIndex()
361     {
362         return this.stopIndex_;
363     }
364 
365     public final void stopIndex(size_t stopIndex)
366     {
367         this.stopIndex_ = stopIndex;
368     }
369 
370 }