1 /*
2  * Copyright (c) 2012-2018 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonToken;
8 
9 import std.conv;
10 import std.array;
11 import std.typecons;
12 import std.container : DList;
13 import antlr.v4.runtime.WritableToken;
14 import antlr.v4.runtime.CharStream;
15 import antlr.v4.runtime.Token;
16 import antlr.v4.runtime.TokenConstantDefinition;
17 import antlr.v4.runtime.TokenSource;
18 import antlr.v4.runtime.misc.Interval;
19 
20 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
21 
22 // Class CommonToken
23 /**
24  * TODO add class description
25  */
26 class CommonToken : WritableToken
27 {
28 
29     /**
30      * An empty {@link Pair} which is used as the default value of
31      * {@link #source} for tokens that do not have a source.
32      */
33     protected static TokenFactorySourcePair EMPTY_SOURCE;
34 
35     /**
36      * This is the backing field for {@link #getType} and {@link #setType}.
37      */
38     protected int type;
39 
40     /**
41      * This is the backing field for {@link #getLine} and {@link #setLine}.
42      */
43     protected int line;
44 
45     /**
46      * This is the backing field for {@link #getCharPositionInLine} and
47      * {@link #setCharPositionInLine}.
48      */
49     protected int charPositionInLine = -1;
50 
51     /**
52      * This is the backing field for {@link #getChannel} and
53      * {@link #setChannel}.
54      */
55     protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL;
56 
57     /**
58      * This is the backing field for {@link #getTokenSource} and
59      * {@link #getInputStream}.
60      *
61      * <p>
62      * These properties share a field to reduce the memory footprint of
63      * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from
64      * the same source and input stream share a reference to the same
65      * {@link Pair} containing these values.</p>
66      */
67     protected TokenFactorySourcePair source;
68 
69     /**
70      * This is the backing field for {@link #getText} when the token text is
71      * explicitly set in the constructor or via {@link #setText}.
72      *
73      *  @see #getText()
74      */
75     protected string text;
76 
77     /**
78      * This is the backing field for {@link #getTokenIndex} and
79      * {@link #setTokenIndex}.
80      */
81     protected int index = -1;
82 
83     /**
84      * This is the backing field for {@link #getStartIndex} and
85      * {@link #setStartIndex}.
86      */
87     protected int start;
88 
89     /**
90      * This is the backing field for {@link #getStopIndex} and
91      * {@link #setStopIndex}.
92      */
93     protected int stop;
94 
95     /**
96      * Constructs a new {@link CommonToken} with the specified token type.
97      *
98      *  @param type The token type.
99      */
100     public this(int type)
101     {
102 	this.type = type;
103         this.source = EMPTY_SOURCE;
104     }
105 
106     public this(TokenFactorySourcePair source, int type, int channel, int start, int stop)
107     {
108 	this.source = source;
109         this.type = type;
110         this.channel = channel;
111         this.start = start;
112         this.stop = stop;
113         if (source.a !is null) {
114             this.line = source.a.getLine();
115             this.charPositionInLine = source.a.getCharPositionInLine();
116         }
117     }
118 
119     /**
120      * Constructs a new {@link CommonToken} with the specified token type and
121      * text.
122      *
123      *  @param type The token type.
124      *  @param text The text of the token.
125      */
126     public this(int type, string text)
127     {
128 	this.type = type;
129         this.channel = TokenConstantDefinition.DEFAULT_CHANNEL;
130         this.text = text;
131         this.source = EMPTY_SOURCE;
132     }
133 
134     /**
135      * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
136      * 	 *
137      * <p>
138      * If {@code oldToken} is also a {@link CommonToken} instance, the newly
139      * constructed token will share a reference to the {@link #text} field and
140      * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will
141      * be assigned the result of calling {@link #getText}, and {@link #source}
142      * will be constructed from the result of {@link Token#getTokenSource} and
143      * {@link Token#getInputStream}.</p>
144      *
145      *  @param oldToken The token to copy.
146      */
147     public this(Token oldToken)
148     {
149 	type = oldToken.getType();
150         line = oldToken.getLine();
151         index = oldToken.getTokenIndex();
152         charPositionInLine = oldToken.getCharPositionInLine();
153         channel = oldToken.getChannel();
154         start = oldToken.getStartIndex();
155         stop = oldToken.getStopIndex();
156 
157         if (oldToken.classinfo == CommonToken.classinfo) {
158             text = (cast(CommonToken)oldToken).text;
159             source = (cast(CommonToken)oldToken).source;
160         }
161         else {
162             text = oldToken.getText();
163             TokenFactorySourcePair sourceNew = tuple(
164                                                      oldToken.getTokenSource,
165                                                      oldToken.getInputStream);
166             source = sourceNew;
167         }
168     }
169 
170     public int getType()
171     {
172         return type;
173     }
174 
175     public void setLine(int line)
176     {
177         this.line = line;
178     }
179 
180     /**
181      * @uml
182      * @override
183      */
184     public override string getText()
185     {
186 	if (text) {
187             return text;
188         }
189 
190         CharStream input = getInputStream();
191         if (input is null) return null;
192         int n = input.size();
193         if ( start<n && stop<n) {
194             return input.getText(Interval.of(start,stop));
195         }
196         else {
197             return "<EOF>";
198         }
199     }
200 
201     /**
202      * @uml
203      * @override
204      * Explicitly set the text for this token. If {code text} is not
205      * {@code null}, then {@link #getText} will return this value rather than
206      * extracting the text from the input.
207      *
208      *  @param text The explicit text of the token, or {@code null} if the text
209      * should be obtained from the input along with the start and stop indexes
210      * of the token.
211      */
212     public override void setText(string text)
213     {
214         this.text = text;
215     }
216 
217     /**
218      * @uml
219      * @override
220      */
221     public override int getLine()
222     {
223         return line;
224     }
225 
226     /**
227      * @uml
228      * @override
229      */
230     public override int getCharPositionInLine()
231     {
232         return charPositionInLine;
233     }
234 
235     public void setCharPositionInLine(int charPositionInLine)
236     {
237         this.charPositionInLine = charPositionInLine;
238     }
239 
240     /**
241      * @uml
242      * @override
243      */
244     public override int getChannel()
245     {
246         return channel;
247     }
248 
249     /**
250      * @uml
251      * @override
252      */
253     public override void setChannel(int channel)
254     {
255         this.channel = channel;
256     }
257 
258     /**
259      * @uml
260      * @override
261      */
262     public override void setType(int type)
263     {
264         this.type = type;
265     }
266 
267     /**
268      * @uml
269      * @override
270      */
271     public override int getStartIndex()
272     {
273         return start;
274     }
275 
276     public void setStartIndex(int start)
277     {
278         this.start = start;
279     }
280 
281     /**
282      * @uml
283      * @override
284      */
285     public override int getStopIndex()
286     {
287         return stop;
288     }
289 
290     public void setStopIndex(int stop)
291     {
292         this.stop = stop;
293     }
294 
295     /**
296      * @uml
297      * @override
298      */
299     public override int getTokenIndex()
300     {
301         return index;
302     }
303 
304     /**
305      * @uml
306      * @override
307      */
308     public override void setTokenIndex(int index)
309     {
310         this.index = index;
311     }
312 
313     /**
314      * @uml
315      * @override
316      */
317     public override TokenSource getTokenSource()
318     {
319         return source.a;
320     }
321 
322     /**
323      * @uml
324      * @override
325      */
326     public override CharStream getInputStream()
327     {
328         return source.b;
329     }
330 
331     /**
332      * @uml
333      * @override
334      */
335     public override string toString()
336     {
337         string channelStr = "";
338         if (channel > 0) {
339             channelStr=",channel=" ~ to!string(channel);
340         }
341         string txt = getText;
342         if (txt) {
343             txt = txt.replace("\n","\\n");
344             txt = txt.replace("\r","\\r");
345             txt = txt.replace("\t","\\t");
346         }
347         else {
348             txt = "<no text>";
349         }
350         return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(start) ~ ":"
351             ~ to!string(stop) ~ "='" ~ txt ~ "',<" ~ to!string(type)
352             ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":"
353             ~ to!string(getCharPositionInLine) ~ "]";
354     }
355 
356 }