1 /*
2  * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonToken;
8 
9 import antlr.v4.runtime.CharStream;
10 import antlr.v4.runtime.Token;
11 import antlr.v4.runtime.TokenConstantDefinition;
12 import antlr.v4.runtime.TokenSource;
13 import antlr.v4.runtime.WritableToken;
14 import antlr.v4.runtime.misc.Interval;
15 import std.array;
16 import std.container : DList;
17 import std.conv;
18 import std.typecons;
19 import std.variant;
20 
21 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
22 
23 /**
24  * TODO add class description
25  */
26 class CommonToken : WritableToken
27 {
28 
29     /**
30      * An empty {@link Pair} which is used as the default value of
31      * {@link #source} for tokens that do not have a source.
32      */
33     protected static TokenFactorySourcePair EMPTY_SOURCE;
34 
35     /**
36      * This is the backing field for {@link #getType} and {@link #setType}.
37      */
38     protected int type;
39 
40     /**
41      * This is the backing field for {@link #getLine} and {@link #setLine}.
42      */
43     protected int line;
44 
45     /**
46      * This is the backing field for {@link #getCharPositionInLine} and
47      * {@link #setCharPositionInLine}.
48      */
49     protected int charPositionInLine = -1;
50 
51     /**
52      * This is the backing field for {@link #getChannel} and
53      * {@link #setChannel}.
54      */
55     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
56 
57     /**
58      * This is the backing field for {@link #getTokenSource} and
59      * {@link #getInputStream}.
60      *
61      * <p>
62      * These properties share a field to reduce the memory footprint of
63      * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
64      * the same source and input stream share a reference to the same
65      * {@link Pair} containing these values.</p>
66      */
67     protected TokenFactorySourcePair source;
68 
69     /**
70      * This is the backing field for {@link #getText} when the token text is
71      * explicitly set in the constructor or via {@link #setText}.
72      *
73      *  @see #getText()
74      */
75     protected Variant text;
76 
77     /**
78      * This is the backing field for {@link #getTokenIndex} and
79      * {@link #setTokenIndex}.
80      */
81     protected int index = -1;
82 
83     /**
84      * This is the backing field for {@link #getStartIndex} and
85      * {@link #setStartIndex}.
86      * @uml
87      * @read
88      * @write
89      */
90     protected int startIndex_;
91 
92     /**
93      * This is the backing field for {@link #getStopIndex} and
94      * {@link #setStopIndex}.
95      * @uml
96      * @read
97      * @write
98      */
99     protected int stopIndex_;
100 
101     /**
102      * Constructs a new {@link CommonToken} with the specified token type.
103      *
104      *  @param type The token type.
105      */
106     public this(int type)
107     {
108 	this.type = type;
109         this.source = EMPTY_SOURCE;
110     }
111 
112     public this(TokenFactorySourcePair source, int type, int channel, int start, int stop)
113     {
114 	this.source = source;
115         this.type = type;
116         this.channel = channel;
117         this.startIndex_ = start;
118         this.stopIndex_ = stop;
119         if (source.a) {
120             this.line = source.a.getLine;
121             this.charPositionInLine = source.a.getCharPositionInLine;
122         }
123     }
124 
125     /**
126      * Constructs a new {@link CommonToken} with the specified token type and
127      * text.
128      *
129      *  @param type The token type.
130      *  @param text The text of the token.
131      */
132     public this(int type, Variant text)
133     {
134 	this.type = type;
135         this.channel = TokenConstantDefinition.DEFAULT_CHANNEL;
136         this.text = text;
137         this.source = EMPTY_SOURCE;
138     }
139 
140     /**
141      * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
142      * 	 *
143      * <p>
144      * If {@code oldToken} is also a {@link CommonToken} instance, the newly
145      * constructed token will share a reference to the {@link #text} field and
146      * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
147      * be assigned the result of calling {@link #getText}, and {@link #source}
148      * will be constructed from the result of {@link Token#getTokenSource} and
149      * {@link Token#getInputStream}.</p>
150      *
151      *  @param oldToken The token to copy.
152      */
153     public this(Token oldToken)
154     {
155 	type = oldToken.getType;
156         line = oldToken.getLine;
157         index = oldToken.getTokenIndex;
158         charPositionInLine = oldToken.getCharPositionInLine;
159         channel = oldToken.getChannel;
160         startIndex_ = oldToken.startIndex;
161         stopIndex_ = oldToken.stopIndex;
162 
163         if (oldToken.classinfo == CommonToken.classinfo) {
164             text = (cast(CommonToken)oldToken).text;
165             source = (cast(CommonToken)oldToken).source;
166         }
167         else {
168             text = oldToken.getText;
169             TokenFactorySourcePair sourceNew = tuple(
170                                                      oldToken.getTokenSource,
171                                                      oldToken.getInputStream);
172             source = sourceNew;
173         }
174     }
175 
176     public int getType()
177     {
178         return type;
179     }
180 
181     public void setLine(int line)
182     {
183         this.line = line;
184     }
185 
186     /**
187      * @uml
188      * @override
189      */
190     public override Variant getText()
191     {
192         Variant Null;
193 	if (text !is Null) {
194             return text;
195         }
196 
197         CharStream input = getInputStream;
198         if (input is null) return Null;
199         int n = input.size;
200         if (startIndex_ < n && stopIndex_ < n) {
201             Variant v = input.getText(Interval.of(startIndex_, stopIndex_));
202             return v;
203         }
204         else {
205             Variant v = "<EOF>";
206             return v;
207         }
208     }
209 
210     /**
211      * Explicitly set the text for this token. If {code text} is not
212      * {@code null}, then {@link #getText} will return this value rather than
213      * extracting the text from the input.
214      *
215      *  @param text The explicit text of the token, or {@code null} if the text
216      * should be obtained from the input along with the start and stop indexes
217      * of the token.
218      * @uml
219      * @override
220      */
221     public override void setText(Variant text)
222     {
223         this.text = text;
224     }
225 
226     /**
227      * @uml
228      * @override
229      */
230     public override int getLine()
231     {
232         return line;
233     }
234 
235     /**
236      * @uml
237      * @override
238      */
239     public override int getCharPositionInLine()
240     {
241         return charPositionInLine;
242     }
243 
244     public void setCharPositionInLine(int charPositionInLine)
245     {
246         this.charPositionInLine = charPositionInLine;
247     }
248 
249     /**
250      * @uml
251      * @override
252      */
253     public override int getChannel()
254     {
255         return channel;
256     }
257 
258     /**
259      * @uml
260      * @override
261      */
262     public override void setChannel(int channel)
263     {
264         this.channel = channel;
265     }
266 
267     /**
268      * @uml
269      * @override
270      */
271     public override void setType(int type)
272     {
273         this.type = type;
274     }
275 
276     /**
277      * @uml
278      * @override
279      */
280     public override int getTokenIndex()
281     {
282         return index;
283     }
284 
285     /**
286      * @uml
287      * @override
288      */
289     public override void setTokenIndex(int index)
290     {
291         this.index = index;
292     }
293 
294     /**
295      * @uml
296      * @override
297      */
298     public override TokenSource getTokenSource()
299     {
300         return source.a;
301     }
302 
303     /**
304      * @uml
305      * @override
306      */
307     public override CharStream getInputStream()
308     {
309         return source.b;
310     }
311 
312     /**
313      * @uml
314      * @override
315      */
316     public override string toString()
317     {
318         string channelStr = "";
319         if (channel > 0) {
320             channelStr=",channel=" ~ to!string(channel);
321         }
322         string txt = to!string(getText);
323         if (txt) {
324             txt = txt.replace("\n","\\n");
325             txt = txt.replace("\r","\\r");
326             txt = txt.replace("\t","\\t");
327         }
328         else {
329             txt = "<no text>";
330         }
331         return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(startIndex_) ~ ":"
332             ~ to!string(stopIndex_) ~ "='" ~ txt ~ "',<" ~ to!string(type)
333             ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":"
334             ~ to!string(getCharPositionInLine) ~ "]";
335     }
336 
337     public final int startIndex()
338     {
339         return this.startIndex_;
340     }
341 
342     public final void startIndex(int startIndex)
343     {
344         this.startIndex_ = startIndex;
345     }
346 
347     public final int stopIndex()
348     {
349         return this.stopIndex_;
350     }
351 
352     public final void stopIndex(int stopIndex)
353     {
354         this.stopIndex_ = stopIndex;
355     }
356 
357 }