1 /* 2 * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 module antlr.v4.runtime.CommonToken; 8 9 import antlr.v4.runtime.CharStream; 10 import antlr.v4.runtime.Token; 11 import antlr.v4.runtime.TokenConstantDefinition; 12 import antlr.v4.runtime.TokenSource; 13 import antlr.v4.runtime.WritableToken; 14 import antlr.v4.runtime.misc.Interval; 15 import std.array; 16 import std.container : DList; 17 import std.conv; 18 import std.typecons; 19 import std.variant; 20 21 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 22 23 /** 24 * TODO add class description 25 */ 26 class CommonToken : WritableToken 27 { 28 29 /** 30 * An empty {@link Pair} which is used as the default value of 31 * {@link #source} for tokens that do not have a source. 32 */ 33 protected static TokenFactorySourcePair EMPTY_SOURCE; 34 35 /** 36 * This is the backing field for {@link #getType} and {@link #setType}. 37 */ 38 protected int type; 39 40 /** 41 * This is the backing field for {@link #getLine} and {@link #setLine}. 42 */ 43 protected int line; 44 45 /** 46 * This is the backing field for {@link #getCharPositionInLine} and 47 * {@link #setCharPositionInLine}. 48 */ 49 protected int charPositionInLine = -1; 50 51 /** 52 * This is the backing field for {@link #getChannel} and 53 * {@link #setChannel}. 54 */ 55 protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL; 56 57 /** 58 * This is the backing field for {@link #getTokenSource} and 59 * {@link #getInputStream}. 60 * 61 * <p> 62 * These properties share a field to reduce the memory footprint of 63 * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from 64 * the same source and input stream share a reference to the same 65 * {@link Pair} containing these values.</p> 66 */ 67 protected TokenFactorySourcePair source; 68 69 /** 70 * This is the backing field for {@link #getText} when the token text is 71 * explicitly set in the constructor or via {@link #setText}. 72 * 73 * @see #getText() 74 */ 75 protected Variant text; 76 77 /** 78 * This is the backing field for {@link #getTokenIndex} and 79 * {@link #setTokenIndex}. 80 */ 81 protected int index = -1; 82 83 /** 84 * This is the backing field for {@link #getStartIndex} and 85 * {@link #setStartIndex}. 86 * @uml 87 * @read 88 * @write 89 */ 90 protected int startIndex_; 91 92 /** 93 * This is the backing field for {@link #getStopIndex} and 94 * {@link #setStopIndex}. 95 * @uml 96 * @read 97 * @write 98 */ 99 protected int stopIndex_; 100 101 /** 102 * Constructs a new {@link CommonToken} with the specified token type. 103 * 104 * @param type The token type. 105 */ 106 public this(int type) 107 { 108 this.type = type; 109 this.source = EMPTY_SOURCE; 110 } 111 112 public this(TokenFactorySourcePair source, int type, int channel, int start, int stop) 113 { 114 this.source = source; 115 this.type = type; 116 this.channel = channel; 117 this.startIndex_ = start; 118 this.stopIndex_ = stop; 119 if (source.a) { 120 this.line = source.a.getLine; 121 this.charPositionInLine = source.a.getCharPositionInLine; 122 } 123 } 124 125 /** 126 * Constructs a new {@link CommonToken} with the specified token type and 127 * text. 128 * 129 * @param type The token type. 130 * @param text The text of the token. 131 */ 132 public this(int type, Variant text) 133 { 134 this.type = type; 135 this.channel = TokenConstantDefinition.DEFAULT_CHANNEL; 136 this.text = text; 137 this.source = EMPTY_SOURCE; 138 } 139 140 /** 141 * Constructs a new {@link CommonToken} as a copy of another {@link Token}. 142 * * 143 * <p> 144 * If {@code oldToken} is also a {@link CommonToken} instance, the newly 145 * constructed token will share a reference to the {@link #text} field and 146 * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will 147 * be assigned the result of calling {@link #getText}, and {@link #source} 148 * will be constructed from the result of {@link Token#getTokenSource} and 149 * {@link Token#getInputStream}.</p> 150 * 151 * @param oldToken The token to copy. 152 */ 153 public this(Token oldToken) 154 { 155 type = oldToken.getType; 156 line = oldToken.getLine; 157 index = oldToken.getTokenIndex; 158 charPositionInLine = oldToken.getCharPositionInLine; 159 channel = oldToken.getChannel; 160 startIndex_ = oldToken.startIndex; 161 stopIndex_ = oldToken.stopIndex; 162 163 if (oldToken.classinfo == CommonToken.classinfo) { 164 text = (cast(CommonToken)oldToken).text; 165 source = (cast(CommonToken)oldToken).source; 166 } 167 else { 168 text = oldToken.getText; 169 TokenFactorySourcePair sourceNew = tuple( 170 oldToken.getTokenSource, 171 oldToken.getInputStream); 172 source = sourceNew; 173 } 174 } 175 176 public int getType() 177 { 178 return type; 179 } 180 181 public void setLine(int line) 182 { 183 this.line = line; 184 } 185 186 /** 187 * @uml 188 * @override 189 */ 190 public override Variant getText() 191 { 192 Variant Null; 193 if (text !is Null) { 194 return text; 195 } 196 197 CharStream input = getInputStream; 198 if (input is null) return Null; 199 int n = input.size; 200 if (startIndex_ < n && stopIndex_ < n) { 201 Variant v = input.getText(Interval.of(startIndex_, stopIndex_)); 202 return v; 203 } 204 else { 205 Variant v = "<EOF>"; 206 return v; 207 } 208 } 209 210 /** 211 * Explicitly set the text for this token. If {code text} is not 212 * {@code null}, then {@link #getText} will return this value rather than 213 * extracting the text from the input. 214 * 215 * @param text The explicit text of the token, or {@code null} if the text 216 * should be obtained from the input along with the start and stop indexes 217 * of the token. 218 * @uml 219 * @override 220 */ 221 public override void setText(Variant text) 222 { 223 this.text = text; 224 } 225 226 /** 227 * @uml 228 * @override 229 */ 230 public override int getLine() 231 { 232 return line; 233 } 234 235 /** 236 * @uml 237 * @override 238 */ 239 public override int getCharPositionInLine() 240 { 241 return charPositionInLine; 242 } 243 244 public void setCharPositionInLine(int charPositionInLine) 245 { 246 this.charPositionInLine = charPositionInLine; 247 } 248 249 /** 250 * @uml 251 * @override 252 */ 253 public override int getChannel() 254 { 255 return channel; 256 } 257 258 /** 259 * @uml 260 * @override 261 */ 262 public override void setChannel(int channel) 263 { 264 this.channel = channel; 265 } 266 267 /** 268 * @uml 269 * @override 270 */ 271 public override void setType(int type) 272 { 273 this.type = type; 274 } 275 276 /** 277 * @uml 278 * @override 279 */ 280 public override int getTokenIndex() 281 { 282 return index; 283 } 284 285 /** 286 * @uml 287 * @override 288 */ 289 public override void setTokenIndex(int index) 290 { 291 this.index = index; 292 } 293 294 /** 295 * @uml 296 * @override 297 */ 298 public override TokenSource getTokenSource() 299 { 300 return source.a; 301 } 302 303 /** 304 * @uml 305 * @override 306 */ 307 public override CharStream getInputStream() 308 { 309 return source.b; 310 } 311 312 /** 313 * @uml 314 * @override 315 */ 316 public override string toString() 317 { 318 string channelStr = ""; 319 if (channel > 0) { 320 channelStr=",channel=" ~ to!string(channel); 321 } 322 string txt = to!string(getText); 323 if (txt) { 324 txt = txt.replace("\n","\\n"); 325 txt = txt.replace("\r","\\r"); 326 txt = txt.replace("\t","\\t"); 327 } 328 else { 329 txt = "<no text>"; 330 } 331 return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(startIndex_) ~ ":" 332 ~ to!string(stopIndex_) ~ "='" ~ txt ~ "',<" ~ to!string(type) 333 ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":" 334 ~ to!string(getCharPositionInLine) ~ "]"; 335 } 336 337 public final int startIndex() 338 { 339 return this.startIndex_; 340 } 341 342 public final void startIndex(int startIndex) 343 { 344 this.startIndex_ = startIndex; 345 } 346 347 public final int stopIndex() 348 { 349 return this.stopIndex_; 350 } 351 352 public final void stopIndex(int stopIndex) 353 { 354 this.stopIndex_ = stopIndex; 355 } 356 357 }