1 /* 2 * Copyright (c) 2012-2020 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 module antlr.v4.runtime.CommonToken; 8 9 import antlr.v4.runtime.CharStream; 10 import antlr.v4.runtime.Token; 11 import antlr.v4.runtime.TokenConstantDefinition; 12 import antlr.v4.runtime.TokenSource; 13 import antlr.v4.runtime.WritableToken; 14 import antlr.v4.runtime.misc.Interval; 15 import std.array; 16 import std.container : DList; 17 import std.conv; 18 import std.typecons; 19 import std.variant; 20 21 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 22 23 /** 24 * TODO add class description 25 */ 26 class CommonToken : WritableToken 27 { 28 29 /** 30 * An empty {@link Pair} which is used as the default value of 31 * {@link #source} for tokens that do not have a source. 32 */ 33 protected static TokenFactorySourcePair EMPTY_SOURCE; 34 35 /** 36 * This is the backing field for {@link #getType} and {@link #setType}. 37 */ 38 protected int type; 39 40 /** 41 * This is the backing field for {@link #getLine} and {@link #setLine}. 42 */ 43 protected int line; 44 45 /** 46 * This is the backing field for {@link #getCharPositionInLine} and 47 * {@link #setCharPositionInLine}. 48 */ 49 protected int charPositionInLine = -1; 50 51 /** 52 * This is the backing field for {@link #getChannel} and 53 * {@link #setChannel}. 54 */ 55 protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL; 56 57 /** 58 * This is the backing field for {@link #getTokenSource} and 59 * {@link #getInputStream}. 60 * 61 * <p> 62 * These properties share a field to reduce the memory footprint of 63 * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from 64 * the same source and input stream share a reference to the same 65 * {@link Pair} containing these values.</p> 66 */ 67 protected TokenFactorySourcePair source; 68 69 /** 70 * This is the backing field for {@link #getText} when the token text is 71 * explicitly set in the constructor or via {@link #setText}. 72 * 73 * @see #getText() 74 */ 75 protected Variant text; 76 77 /** 78 * This is the backing field for {@link #getTokenIndex} and 79 * {@link #setTokenIndex}. 80 */ 81 protected size_t index = size_t.max; 82 83 /** 84 * This is the backing field for {@link #getStartIndex} and 85 * {@link #setStartIndex}. 86 * @uml 87 * @read 88 * @write 89 */ 90 protected size_t startIndex_; 91 92 /** 93 * This is the backing field for {@link #getStopIndex} and 94 * {@link #setStopIndex}. 95 * @uml 96 * @read 97 * @write 98 */ 99 protected size_t stopIndex_; 100 101 /** 102 * Constructs a new {@link CommonToken} with the specified token type. 103 * 104 * @param type The token type. 105 */ 106 public this(int type) 107 { 108 this.type = type; 109 this.source = EMPTY_SOURCE; 110 } 111 112 public this(TokenFactorySourcePair source, int type, int channel, size_t start, size_t stop) 113 { 114 this.source = source; 115 this.type = type; 116 this.channel = channel; 117 this.startIndex_ = start; 118 this.stopIndex_ = stop; 119 if (source.a) { 120 this.line = source.a.getLine; 121 this.charPositionInLine = source.a.getCharPositionInLine; 122 } 123 } 124 125 /** 126 * Constructs a new {@link CommonToken} with the specified token type and 127 * text. 128 * 129 * @param type The token type. 130 * @param text The text of the token. 131 */ 132 public this(int type, Variant text) 133 { 134 this.type = type; 135 this.channel = TokenConstantDefinition.DEFAULT_CHANNEL; 136 this.text = text; 137 this.source = EMPTY_SOURCE; 138 } 139 140 /** 141 * Constructs a new {@link CommonToken} as a copy of another {@link Token}. 142 * * 143 * <p> 144 * If {@code oldToken} is also a {@link CommonToken} instance, the newly 145 * constructed token will share a reference to the {@link #text} field and 146 * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will 147 * be assigned the result of calling {@link #getText}, and {@link #source} 148 * will be constructed from the result of {@link Token#getTokenSource} and 149 * {@link Token#getInputStream}.</p> 150 * 151 * @param oldToken The token to copy. 152 */ 153 public this(Token oldToken) 154 { 155 type = oldToken.getType; 156 line = oldToken.getLine; 157 index = oldToken.getTokenIndex; 158 charPositionInLine = oldToken.getCharPositionInLine; 159 channel = oldToken.getChannel; 160 startIndex_ = oldToken.startIndex; 161 stopIndex_ = oldToken.stopIndex; 162 163 if (cast(CommonToken)oldToken) { 164 text = (cast(CommonToken)oldToken).text; 165 source = (cast(CommonToken)oldToken).source; 166 } 167 else { 168 text = oldToken.getText; 169 TokenFactorySourcePair sourceNew = tuple( 170 oldToken.getTokenSource, 171 oldToken.getInputStream); 172 source = sourceNew; 173 } 174 } 175 176 public int getType() 177 { 178 return type; 179 } 180 181 public void setLine(int line) 182 { 183 this.line = line; 184 } 185 186 /** 187 * @uml 188 * @override 189 */ 190 public override Variant getText() 191 { 192 Variant Null; 193 if (text !is Null) 194 { 195 return text; 196 } 197 198 CharStream input = getInputStream; 199 if (input is null) return Null; 200 auto n = input.size; 201 if (startIndex_ < n && stopIndex_ < n) { 202 Variant v = input.getText(Interval.of(to!int(startIndex_), to!int(stopIndex_))); 203 return v; 204 } 205 else { 206 Variant v = "<EOF>"; 207 return v; 208 } 209 } 210 211 /** 212 * Explicitly set the text for this token. If {code text} is not 213 * {@code null}, then {@link #getText} will return this value rather than 214 * extracting the text from the input. 215 * 216 * @param text The explicit text of the token, or {@code null} if the text 217 * should be obtained from the input along with the start and stop indexes 218 * of the token. 219 * @uml 220 * @override 221 */ 222 public override void setText(Variant text) 223 { 224 this.text = text; 225 } 226 227 /** 228 * @uml 229 * @override 230 */ 231 public override int getLine() 232 { 233 return line; 234 } 235 236 /** 237 * @uml 238 * @override 239 */ 240 public override int getCharPositionInLine() 241 { 242 return charPositionInLine; 243 } 244 245 public void setCharPositionInLine(int charPositionInLine) 246 { 247 this.charPositionInLine = charPositionInLine; 248 } 249 250 /** 251 * @uml 252 * @override 253 */ 254 public override int getChannel() 255 { 256 return channel; 257 } 258 259 /** 260 * @uml 261 * @override 262 */ 263 public override void setChannel(int channel) 264 { 265 this.channel = channel; 266 } 267 268 /** 269 * @uml 270 * @override 271 */ 272 public override void setType(int type) 273 { 274 this.type = type; 275 } 276 277 /** 278 * @uml 279 * @override 280 */ 281 public override size_t getTokenIndex() 282 { 283 return index; 284 } 285 286 /** 287 * @uml 288 * @override 289 */ 290 public override void setTokenIndex(size_t index) 291 { 292 this.index = index; 293 } 294 295 /** 296 * @uml 297 * @override 298 */ 299 public override TokenSource getTokenSource() 300 { 301 return source.a; 302 } 303 304 /** 305 * @uml 306 * @override 307 */ 308 public override CharStream getInputStream() 309 { 310 return source.b; 311 } 312 313 /** 314 * @uml 315 * @override 316 */ 317 public override string toString() 318 { 319 import std.format : format; 320 321 string channelStr = ""; 322 if (channel > 0) { 323 channelStr=",channel=" ~ to!string(channel); 324 } 325 auto txt = getText.get!(string); 326 if (txt.length > 0) { 327 txt = txt.replace("\n","\\n"); 328 txt = txt.replace("\r","\\r"); 329 txt = txt.replace("\t","\\t"); 330 } 331 else { 332 txt = "<no text>"; 333 } 334 return format!"[@%s,%s:%s='%s',<%s>%s,%s:%s]"(cast(int)getTokenIndex, 335 cast(int)startIndex_, 336 cast(int)stopIndex_, 337 txt, type, channelStr, 338 line, getCharPositionInLine 339 ); 340 } 341 342 public final size_t startIndex() 343 { 344 return this.startIndex_; 345 } 346 347 public final void startIndex(size_t startIndex) 348 { 349 this.startIndex_ = startIndex; 350 } 351 352 public final size_t stopIndex() 353 { 354 return this.stopIndex_; 355 } 356 357 public final void stopIndex(size_t stopIndex) 358 { 359 this.stopIndex_ = stopIndex; 360 } 361 362 }