1 /*
2  * Copyright (c) 2012-2020 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonToken;
8 
9 import antlr.v4.runtime.CharStream;
10 import antlr.v4.runtime.Token;
11 import antlr.v4.runtime.TokenConstantDefinition;
12 import antlr.v4.runtime.TokenSource;
13 import antlr.v4.runtime.WritableToken;
14 import antlr.v4.runtime.misc.Interval;
15 import std.array;
16 import std.container : DList;
17 import std.conv;
18 import std.typecons;
19 import std.variant;
20 
21 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
22 
23 /**
24  * TODO add class description
25  */
26 class CommonToken : WritableToken
27 {
28 
29     /**
30      * An empty {@link Pair} which is used as the default value of
31      * {@link #source} for tokens that do not have a source.
32      */
33     protected static TokenFactorySourcePair EMPTY_SOURCE;
34 
35     /**
36      * This is the backing field for {@link #getType} and {@link #setType}.
37      */
38     protected int type;
39 
40     /**
41      * This is the backing field for {@link #getLine} and {@link #setLine}.
42      */
43     protected int line;
44 
45     /**
46      * This is the backing field for {@link #getCharPositionInLine} and
47      * {@link #setCharPositionInLine}.
48      */
49     protected int charPositionInLine = -1;
50 
51     /**
52      * This is the backing field for {@link #getChannel} and
53      * {@link #setChannel}.
54      */
55     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
56 
57     /**
58      * This is the backing field for {@link #getTokenSource} and
59      * {@link #getInputStream}.
60      *
61      * <p>
62      * These properties share a field to reduce the memory footprint of
63      * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
64      * the same source and input stream share a reference to the same
65      * {@link Pair} containing these values.</p>
66      */
67     protected TokenFactorySourcePair source;
68 
69     /**
70      * This is the backing field for {@link #getText} when the token text is
71      * explicitly set in the constructor or via {@link #setText}.
72      *
73      *  @see #getText()
74      */
75     protected Variant text;
76 
77     /**
78      * This is the backing field for {@link #getTokenIndex} and
79      * {@link #setTokenIndex}.
80      */
81     protected size_t index = size_t.max;
82 
83     /**
84      * This is the backing field for {@link #getStartIndex} and
85      * {@link #setStartIndex}.
86      * @uml
87      * @read
88      * @write
89      */
90     protected size_t startIndex_;
91 
92     /**
93      * This is the backing field for {@link #getStopIndex} and
94      * {@link #setStopIndex}.
95      * @uml
96      * @read
97      * @write
98      */
99     protected size_t stopIndex_;
100 
101     /**
102      * Constructs a new {@link CommonToken} with the specified token type.
103      *
104      *  @param type The token type.
105      */
106     public this(int type)
107     {
108         this.type = type;
109         this.source = EMPTY_SOURCE;
110     }
111 
112     public this(TokenFactorySourcePair source, int type, int channel, size_t start, size_t stop)
113     {
114     this.source = source;
115         this.type = type;
116         this.channel = channel;
117         this.startIndex_ = start;
118         this.stopIndex_ = stop;
119         if (source.a) {
120             this.line = source.a.getLine;
121             this.charPositionInLine = source.a.getCharPositionInLine;
122         }
123     }
124 
125     /**
126      * Constructs a new {@link CommonToken} with the specified token type and
127      * text.
128      *
129      *  @param type The token type.
130      *  @param text The text of the token.
131      */
132     public this(int type, Variant text)
133     {
134     this.type = type;
135         this.channel = TokenConstantDefinition.DEFAULT_CHANNEL;
136         this.text = text;
137         this.source = EMPTY_SOURCE;
138     }
139 
140     /**
141      * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
142      *   *
143      * <p>
144      * If {@code oldToken} is also a {@link CommonToken} instance, the newly
145      * constructed token will share a reference to the {@link #text} field and
146      * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
147      * be assigned the result of calling {@link #getText}, and {@link #source}
148      * will be constructed from the result of {@link Token#getTokenSource} and
149      * {@link Token#getInputStream}.</p>
150      *
151      *  @param oldToken The token to copy.
152      */
153     public this(Token oldToken)
154     {
155     type = oldToken.getType;
156         line = oldToken.getLine;
157         index = oldToken.getTokenIndex;
158         charPositionInLine = oldToken.getCharPositionInLine;
159         channel = oldToken.getChannel;
160         startIndex_ = oldToken.startIndex;
161         stopIndex_ = oldToken.stopIndex;
162 
163         if (cast(CommonToken)oldToken) {
164             text = (cast(CommonToken)oldToken).text;
165             source = (cast(CommonToken)oldToken).source;
166         }
167         else {
168             text = oldToken.getText;
169             TokenFactorySourcePair sourceNew = tuple(
170                                                      oldToken.getTokenSource,
171                                                      oldToken.getInputStream);
172             source = sourceNew;
173         }
174     }
175 
176     public int getType()
177     {
178         return type;
179     }
180 
181     public void setLine(int line)
182     {
183         this.line = line;
184     }
185 
186     /**
187      * @uml
188      * @override
189      */
190     public override Variant getText()
191     {
192         Variant Null;
193         if (text !is Null)
194         {
195             return text;
196         }
197 
198         CharStream input = getInputStream;
199         if (input is null) return Null;
200         auto n = input.size;
201         if (startIndex_ < n && stopIndex_ < n) {
202             Variant v = input.getText(Interval.of(to!int(startIndex_), to!int(stopIndex_)));
203             return v;
204         }
205         else {
206             Variant v = "<EOF>";
207             return v;
208         }
209     }
210 
211     /**
212      * Explicitly set the text for this token. If {code text} is not
213      * {@code null}, then {@link #getText} will return this value rather than
214      * extracting the text from the input.
215      *
216      *  @param text The explicit text of the token, or {@code null} if the text
217      * should be obtained from the input along with the start and stop indexes
218      * of the token.
219      * @uml
220      * @override
221      */
222     public override void setText(Variant text)
223     {
224         this.text = text;
225     }
226 
227     /**
228      * @uml
229      * @override
230      */
231     public override int getLine()
232     {
233         return line;
234     }
235 
236     /**
237      * @uml
238      * @override
239      */
240     public override int getCharPositionInLine()
241     {
242         return charPositionInLine;
243     }
244 
245     public void setCharPositionInLine(int charPositionInLine)
246     {
247         this.charPositionInLine = charPositionInLine;
248     }
249 
250     /**
251      * @uml
252      * @override
253      */
254     public override int getChannel()
255     {
256         return channel;
257     }
258 
259     /**
260      * @uml
261      * @override
262      */
263     public override void setChannel(int channel)
264     {
265         this.channel = channel;
266     }
267 
268     /**
269      * @uml
270      * @override
271      */
272     public override void setType(int type)
273     {
274         this.type = type;
275     }
276 
277     /**
278      * @uml
279      * @override
280      */
281     public override size_t getTokenIndex()
282     {
283         return index;
284     }
285 
286     /**
287      * @uml
288      * @override
289      */
290     public override void setTokenIndex(size_t index)
291     {
292         this.index = index;
293     }
294 
295     /**
296      * @uml
297      * @override
298      */
299     public override TokenSource getTokenSource()
300     {
301         return source.a;
302     }
303 
304     /**
305      * @uml
306      * @override
307      */
308     public override CharStream getInputStream()
309     {
310         return source.b;
311     }
312 
313     /**
314      * @uml
315      * @override
316      */
317     public override string toString()
318     {
319         import std.format : format;
320 
321         string channelStr = "";
322         if (channel > 0) {
323             channelStr=",channel=" ~ to!string(channel);
324         }
325         auto txt = getText.get!(string);
326         if (txt.length > 0) {
327             txt = txt.replace("\n","\\n");
328             txt = txt.replace("\r","\\r");
329             txt = txt.replace("\t","\\t");
330         }
331         else {
332             txt = "<no text>";
333         }
334         return format!"[@%s,%s:%s='%s',<%s>%s,%s:%s]"(cast(int)getTokenIndex,
335             cast(int)startIndex_,
336             cast(int)stopIndex_,
337             txt, type, channelStr,
338             line, getCharPositionInLine
339             );
340     }
341 
342     public final size_t startIndex()
343     {
344         return this.startIndex_;
345     }
346 
347     public final void startIndex(size_t startIndex)
348     {
349         this.startIndex_ = startIndex;
350     }
351 
352     public final size_t stopIndex()
353     {
354         return this.stopIndex_;
355     }
356 
357     public final void stopIndex(size_t stopIndex)
358     {
359         this.stopIndex_ = stopIndex;
360     }
361 
362 }