1 /* 2 * Copyright (c) 2012-2020 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 module antlr.v4.runtime.CommonToken; 8 9 import antlr.v4.runtime.CharStream; 10 import antlr.v4.runtime.Token; 11 import antlr.v4.runtime.TokenConstantDefinition; 12 import antlr.v4.runtime.TokenSource; 13 import antlr.v4.runtime.WritableToken; 14 import antlr.v4.runtime.misc.Interval; 15 import std.array; 16 import std.container : DList; 17 import std.conv; 18 import std.typecons; 19 import std.variant; 20 21 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 22 23 /** 24 * TODO add class description 25 */ 26 class CommonToken : WritableToken 27 { 28 29 /** 30 * An empty {@link Pair} which is used as the default value of 31 * {@link #source} for tokens that do not have a source. 32 */ 33 protected static TokenFactorySourcePair EMPTY_SOURCE; 34 35 /** 36 * This is the backing field for {@link #getType} and {@link #setType}. 37 */ 38 protected int type; 39 40 /** 41 * This is the backing field for {@link #getLine} and {@link #setLine}. 42 */ 43 protected int line; 44 45 /** 46 * This is the backing field for {@link #getCharPositionInLine} and 47 * {@link #setCharPositionInLine}. 48 */ 49 protected int charPositionInLine = -1; 50 51 /** 52 * This is the backing field for {@link #getChannel} and 53 * {@link #setChannel}. 54 */ 55 protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL; 56 57 /** 58 * This is the backing field for {@link #getTokenSource} and 59 * {@link #getInputStream}. 60 * 61 * <p> 62 * These properties share a field to reduce the memory footprint of 63 * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from 64 * the same source and input stream share a reference to the same 65 * {@link Pair} containing these values.</p> 66 */ 67 protected TokenFactorySourcePair source; 68 69 /** 70 * This is the backing field for {@link #getText} when the token text is 71 * explicitly set in the constructor or via {@link #setText}. 72 * 73 * @see #getText() 74 */ 75 protected Variant text; 76 77 /** 78 * This is the backing field for {@link #getTokenIndex} and 79 * {@link #setTokenIndex}. 80 */ 81 protected size_t index = size_t.max; 82 83 /** 84 * This is the backing field for {@link #getStartIndex} and 85 * {@link #setStartIndex}. 86 * @uml 87 * @read 88 * @write 89 */ 90 protected size_t startIndex_; 91 92 /** 93 * This is the backing field for {@link #getStopIndex} and 94 * {@link #setStopIndex}. 95 * @uml 96 * @read 97 * @write 98 */ 99 protected size_t stopIndex_; 100 101 /** 102 * Constructs a new {@link CommonToken} with the specified token type. 103 * 104 * @param type The token type. 105 */ 106 public this(int type) 107 { 108 this.type = type; 109 this.source = EMPTY_SOURCE; 110 } 111 112 public this(TokenFactorySourcePair source, int type, int channel, size_t start, size_t stop) 113 { 114 this.source = source; 115 this.type = type; 116 this.channel = channel; 117 this.startIndex_ = start; 118 this.stopIndex_ = stop; 119 if (source.a) 120 { 121 this.line = source.a.getLine; 122 this.charPositionInLine = source.a.getCharPositionInLine; 123 } 124 } 125 126 /** 127 * Constructs a new {@link CommonToken} with the specified token type and 128 * text. 129 * 130 * @param type The token type. 131 * @param text The text of the token. 132 */ 133 public this(int type, Variant text) 134 { 135 this.type = type; 136 this.channel = TokenConstantDefinition.DEFAULT_CHANNEL; 137 this.text = text; 138 this.source = EMPTY_SOURCE; 139 } 140 141 /** 142 * Constructs a new {@link CommonToken} as a copy of another {@link Token}. 143 * * 144 * <p> 145 * If {@code oldToken} is also a {@link CommonToken} instance, the newly 146 * constructed token will share a reference to the {@link #text} field and 147 * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will 148 * be assigned the result of calling {@link #getText}, and {@link #source} 149 * will be constructed from the result of {@link Token#getTokenSource} and 150 * {@link Token#getInputStream}.</p> 151 * 152 * @param oldToken The token to copy. 153 */ 154 public this(Token oldToken) 155 { 156 type = oldToken.getType; 157 line = oldToken.getLine; 158 index = oldToken.getTokenIndex; 159 charPositionInLine = oldToken.getCharPositionInLine; 160 channel = oldToken.getChannel; 161 startIndex_ = oldToken.startIndex; 162 stopIndex_ = oldToken.stopIndex; 163 164 if (cast(CommonToken) oldToken) 165 { 166 text = (cast(CommonToken) oldToken).text; 167 source = (cast(CommonToken) oldToken).source; 168 } 169 else 170 { 171 text = oldToken.getText; 172 TokenFactorySourcePair sourceNew = tuple(oldToken.getTokenSource, 173 oldToken.getInputStream); 174 source = sourceNew; 175 } 176 } 177 178 public int getType() 179 { 180 return type; 181 } 182 183 public void setLine(int line) 184 { 185 this.line = line; 186 } 187 188 /** 189 * @uml 190 * @override 191 */ 192 public override Variant getText() 193 { 194 Variant Null; 195 if (text !is Null) 196 { 197 return text; 198 } 199 200 CharStream input = getInputStream; 201 if (input is null) 202 return Null; 203 auto n = input.size; 204 if (startIndex_ < n && stopIndex_ < n) 205 { 206 Variant v = input.getText(Interval.of(to!int(startIndex_), to!int(stopIndex_))); 207 return v; 208 } 209 else 210 { 211 Variant v = "<EOF>"; 212 return v; 213 } 214 } 215 216 /** 217 * Explicitly set the text for this token. If {code text} is not 218 * {@code null}, then {@link #getText} will return this value rather than 219 * extracting the text from the input. 220 * 221 * @param text The explicit text of the token, or {@code null} if the text 222 * should be obtained from the input along with the start and stop indexes 223 * of the token. 224 * @uml 225 * @override 226 */ 227 public override void setText(Variant text) 228 { 229 this.text = text; 230 } 231 232 /** 233 * @uml 234 * @override 235 */ 236 public override int getLine() 237 { 238 return line; 239 } 240 241 /** 242 * @uml 243 * @override 244 */ 245 public override int getCharPositionInLine() 246 { 247 return charPositionInLine; 248 } 249 250 public void setCharPositionInLine(int charPositionInLine) 251 { 252 this.charPositionInLine = charPositionInLine; 253 } 254 255 /** 256 * @uml 257 * @override 258 */ 259 public override int getChannel() 260 { 261 return channel; 262 } 263 264 /** 265 * @uml 266 * @override 267 */ 268 public override void setChannel(int channel) 269 { 270 this.channel = channel; 271 } 272 273 /** 274 * @uml 275 * @override 276 */ 277 public override void setType(int type) 278 { 279 this.type = type; 280 } 281 282 /** 283 * @uml 284 * @override 285 */ 286 public override size_t getTokenIndex() 287 { 288 return index; 289 } 290 291 /** 292 * @uml 293 * @override 294 */ 295 public override void setTokenIndex(size_t index) 296 { 297 this.index = index; 298 } 299 300 /** 301 * @uml 302 * @override 303 */ 304 public override TokenSource getTokenSource() 305 { 306 return source.a; 307 } 308 309 /** 310 * @uml 311 * @override 312 */ 313 public override CharStream getInputStream() 314 { 315 return source.b; 316 } 317 318 /** 319 * @uml 320 * @override 321 */ 322 public override string toString() 323 { 324 import std.format : format; 325 326 string channelStr = ""; 327 if (channel > 0) 328 { 329 channelStr = ",channel=" ~ to!string(channel); 330 } 331 auto txt = getText.get!(string); 332 if (txt.length > 0) 333 { 334 txt = txt.replace("\n", "\\n"); 335 txt = txt.replace("\r", "\\r"); 336 txt = txt.replace("\t", "\\t"); 337 } 338 else 339 { 340 txt = "<no text>"; 341 } 342 return format!"[@%s,%s:%s='%s',<%s>%s,%s:%s]"(cast(int)getTokenIndex, 343 cast(int)startIndex_, 344 cast(int)stopIndex_, 345 txt, type, channelStr, 346 line, getCharPositionInLine 347 ); 348 } 349 350 public final size_t startIndex() 351 { 352 return this.startIndex_; 353 } 354 355 public final void startIndex(size_t startIndex) 356 { 357 this.startIndex_ = startIndex; 358 } 359 360 public final size_t stopIndex() 361 { 362 return this.stopIndex_; 363 } 364 365 public final void stopIndex(size_t stopIndex) 366 { 367 this.stopIndex_ = stopIndex; 368 } 369 370 }