1 /*
2  * Copyright (c) 2012-2018 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonToken;
8 
9 import std.conv;
10 import std.array;
11 import std.typecons;
12 import std.container : DList;
13 import antlr.v4.runtime.WritableToken;
14 import antlr.v4.runtime.CharStream;
15 import antlr.v4.runtime.Token;
16 import antlr.v4.runtime.TokenConstantDefinition;
17 import antlr.v4.runtime.TokenSource;
18 import antlr.v4.runtime.misc.Interval;
19 
20 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
21 
22 /**
23  * TODO add class description
24  */
25 class CommonToken : WritableToken
26 {
27 
28     /**
29      * An empty {@link Pair} which is used as the default value of
30      * {@link #source} for tokens that do not have a source.
31      */
32     protected static TokenFactorySourcePair EMPTY_SOURCE;
33 
34     /**
35      * This is the backing field for {@link #getType} and {@link #setType}.
36      */
37     protected int type;
38 
39     /**
40      * This is the backing field for {@link #getLine} and {@link #setLine}.
41      */
42     protected int line;
43 
44     /**
45      * This is the backing field for {@link #getCharPositionInLine} and
46      * {@link #setCharPositionInLine}.
47      */
48     protected int charPositionInLine = -1;
49 
50     /**
51      * This is the backing field for {@link #getChannel} and
52      * {@link #setChannel}.
53      */
54     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
55 
56     /**
57      * This is the backing field for {@link #getTokenSource} and
58      * {@link #getInputStream}.
59      *
60      * <p>
61      * These properties share a field to reduce the memory footprint of
62      * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
63      * the same source and input stream share a reference to the same
64      * {@link Pair} containing these values.</p>
65      */
66     protected TokenFactorySourcePair source;
67 
68     /**
69      * This is the backing field for {@link #getText} when the token text is
70      * explicitly set in the constructor or via {@link #setText}.
71      *
72      *  @see #getText()
73      */
74     protected string text;
75 
76     /**
77      * This is the backing field for {@link #getTokenIndex} and
78      * {@link #setTokenIndex}.
79      */
80     protected int index = -1;
81 
82     /**
83      * This is the backing field for {@link #getStartIndex} and
84      * {@link #setStartIndex}.
85      */
86     protected int start;
87 
88     /**
89      * This is the backing field for {@link #getStopIndex} and
90      * {@link #setStopIndex}.
91      */
92     protected int stop;
93 
94     /**
95      * Constructs a new {@link CommonToken} with the specified token type.
96      *
97      *  @param type The token type.
98      */
99     public this(int type)
100     {
101 	this.type = type;
102         this.source = EMPTY_SOURCE;
103     }
104 
105     public this(TokenFactorySourcePair source, int type, int channel, int start, int stop)
106     {
107 	this.source = source;
108         this.type = type;
109         this.channel = channel;
110         this.start = start;
111         this.stop = stop;
112         if (source.a !is null) {
113             this.line = source.a.getLine();
114             this.charPositionInLine = source.a.getCharPositionInLine();
115         }
116     }
117 
118     /**
119      * Constructs a new {@link CommonToken} with the specified token type and
120      * text.
121      *
122      *  @param type The token type.
123      *  @param text The text of the token.
124      */
125     public this(int type, string text)
126     {
127 	this.type = type;
128         this.channel = TokenConstantDefinition.DEFAULT_CHANNEL;
129         this.text = text;
130         this.source = EMPTY_SOURCE;
131     }
132 
133     /**
134      * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
135      * 	 *
136      * <p>
137      * If {@code oldToken} is also a {@link CommonToken} instance, the newly
138      * constructed token will share a reference to the {@link #text} field and
139      * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
140      * be assigned the result of calling {@link #getText}, and {@link #source}
141      * will be constructed from the result of {@link Token#getTokenSource} and
142      * {@link Token#getInputStream}.</p>
143      *
144      *  @param oldToken The token to copy.
145      */
146     public this(Token oldToken)
147     {
148 	type = oldToken.getType();
149         line = oldToken.getLine();
150         index = oldToken.getTokenIndex();
151         charPositionInLine = oldToken.getCharPositionInLine();
152         channel = oldToken.getChannel();
153         start = oldToken.getStartIndex();
154         stop = oldToken.getStopIndex();
155 
156         if (oldToken.classinfo == CommonToken.classinfo) {
157             text = (cast(CommonToken)oldToken).text;
158             source = (cast(CommonToken)oldToken).source;
159         }
160         else {
161             text = oldToken.getText();
162             TokenFactorySourcePair sourceNew = tuple(
163                                                      oldToken.getTokenSource,
164                                                      oldToken.getInputStream);
165             source = sourceNew;
166         }
167     }
168 
169     public int getType()
170     {
171         return type;
172     }
173 
174     public void setLine(int line)
175     {
176         this.line = line;
177     }
178 
179     /**
180      * @uml
181      * @override
182      */
183     public override string getText()
184     {
185 	if (text) {
186             return text;
187         }
188 
189         CharStream input = getInputStream();
190         if (input is null) return null;
191         int n = input.size();
192         if ( start<n && stop<n) {
193             return input.getText(Interval.of(start,stop));
194         }
195         else {
196             return "<EOF>";
197         }
198     }
199 
200     /**
201      * @uml
202      * @override
203      * Explicitly set the text for this token. If {code text} is not
204      * {@code null}, then {@link #getText} will return this value rather than
205      * extracting the text from the input.
206      *
207      *  @param text The explicit text of the token, or {@code null} if the text
208      * should be obtained from the input along with the start and stop indexes
209      * of the token.
210      */
211     public override void setText(string text)
212     {
213         this.text = text;
214     }
215 
216     /**
217      * @uml
218      * @override
219      */
220     public override int getLine()
221     {
222         return line;
223     }
224 
225     /**
226      * @uml
227      * @override
228      */
229     public override int getCharPositionInLine()
230     {
231         return charPositionInLine;
232     }
233 
234     public void setCharPositionInLine(int charPositionInLine)
235     {
236         this.charPositionInLine = charPositionInLine;
237     }
238 
239     /**
240      * @uml
241      * @override
242      */
243     public override int getChannel()
244     {
245         return channel;
246     }
247 
248     /**
249      * @uml
250      * @override
251      */
252     public override void setChannel(int channel)
253     {
254         this.channel = channel;
255     }
256 
257     /**
258      * @uml
259      * @override
260      */
261     public override void setType(int type)
262     {
263         this.type = type;
264     }
265 
266     /**
267      * @uml
268      * @override
269      */
270     public override int getStartIndex()
271     {
272         return start;
273     }
274 
275     public void setStartIndex(int start)
276     {
277         this.start = start;
278     }
279 
280     /**
281      * @uml
282      * @override
283      */
284     public override int getStopIndex()
285     {
286         return stop;
287     }
288 
289     public void setStopIndex(int stop)
290     {
291         this.stop = stop;
292     }
293 
294     /**
295      * @uml
296      * @override
297      */
298     public override int getTokenIndex()
299     {
300         return index;
301     }
302 
303     /**
304      * @uml
305      * @override
306      */
307     public override void setTokenIndex(int index)
308     {
309         this.index = index;
310     }
311 
312     /**
313      * @uml
314      * @override
315      */
316     public override TokenSource getTokenSource()
317     {
318         return source.a;
319     }
320 
321     /**
322      * @uml
323      * @override
324      */
325     public override CharStream getInputStream()
326     {
327         return source.b;
328     }
329 
330     /**
331      * @uml
332      * @override
333      */
334     public override string toString()
335     {
336         string channelStr = "";
337         if (channel > 0) {
338             channelStr=",channel=" ~ to!string(channel);
339         }
340         string txt = getText;
341         if (txt) {
342             txt = txt.replace("\n","\\n");
343             txt = txt.replace("\r","\\r");
344             txt = txt.replace("\t","\\t");
345         }
346         else {
347             txt = "<no text>";
348         }
349         return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(start) ~ ":"
350             ~ to!string(stop) ~ "='" ~ txt ~ "',<" ~ to!string(type)
351             ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":"
352             ~ to!string(getCharPositionInLine) ~ "]";
353     }
354 
355 }