1 /* 2 * Copyright (c) 2012-2018 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 module antlr.v4.runtime.CommonToken; 8 9 import std.conv; 10 import std.array; 11 import std.typecons; 12 import std.container : DList; 13 import antlr.v4.runtime.WritableToken; 14 import antlr.v4.runtime.CharStream; 15 import antlr.v4.runtime.Token; 16 import antlr.v4.runtime.TokenConstantDefinition; 17 import antlr.v4.runtime.TokenSource; 18 import antlr.v4.runtime.misc.Interval; 19 20 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 21 22 // Class CommonToken 23 /** 24 * TODO add class description 25 */ 26 class CommonToken : WritableToken 27 { 28 29 /** 30 * An empty {@link Pair} which is used as the default value of 31 * {@link #source} for tokens that do not have a source. 32 */ 33 protected static TokenFactorySourcePair EMPTY_SOURCE; 34 35 /** 36 * This is the backing field for {@link #getType} and {@link #setType}. 37 */ 38 protected int type; 39 40 /** 41 * This is the backing field for {@link #getLine} and {@link #setLine}. 42 */ 43 protected int line; 44 45 /** 46 * This is the backing field for {@link #getCharPositionInLine} and 47 * {@link #setCharPositionInLine}. 48 */ 49 protected int charPositionInLine = -1; 50 51 /** 52 * This is the backing field for {@link #getChannel} and 53 * {@link #setChannel}. 54 */ 55 protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL; 56 57 /** 58 * This is the backing field for {@link #getTokenSource} and 59 * {@link #getInputStream}. 60 * 61 * <p> 62 * These properties share a field to reduce the memory footprint of 63 * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from 64 * the same source and input stream share a reference to the same 65 * {@link Pair} containing these values.</p> 66 */ 67 protected TokenFactorySourcePair source; 68 69 /** 70 * This is the backing field for {@link #getText} when the token text is 71 * explicitly set in the constructor or via {@link #setText}. 72 * 73 * @see #getText() 74 */ 75 protected string text; 76 77 /** 78 * This is the backing field for {@link #getTokenIndex} and 79 * {@link #setTokenIndex}. 80 */ 81 protected int index = -1; 82 83 /** 84 * This is the backing field for {@link #getStartIndex} and 85 * {@link #setStartIndex}. 86 */ 87 protected int start; 88 89 /** 90 * This is the backing field for {@link #getStopIndex} and 91 * {@link #setStopIndex}. 92 */ 93 protected int stop; 94 95 /** 96 * Constructs a new {@link CommonToken} with the specified token type. 97 * 98 * @param type The token type. 99 */ 100 public this(int type) 101 { 102 this.type = type; 103 this.source = EMPTY_SOURCE; 104 } 105 106 public this(TokenFactorySourcePair source, int type, int channel, int start, int stop) 107 { 108 this.source = source; 109 this.type = type; 110 this.channel = channel; 111 this.start = start; 112 this.stop = stop; 113 if (source.a !is null) { 114 this.line = source.a.getLine(); 115 this.charPositionInLine = source.a.getCharPositionInLine(); 116 } 117 } 118 119 /** 120 * Constructs a new {@link CommonToken} with the specified token type and 121 * text. 122 * 123 * @param type The token type. 124 * @param text The text of the token. 125 */ 126 public this(int type, string text) 127 { 128 this.type = type; 129 this.channel = TokenConstantDefinition.DEFAULT_CHANNEL; 130 this.text = text; 131 this.source = EMPTY_SOURCE; 132 } 133 134 /** 135 * Constructs a new {@link CommonToken} as a copy of another {@link Token}. 136 * * 137 * <p> 138 * If {@code oldToken} is also a {@link CommonToken} instance, the newly 139 * constructed token will share a reference to the {@link #text} field and 140 * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will 141 * be assigned the result of calling {@link #getText}, and {@link #source} 142 * will be constructed from the result of {@link Token#getTokenSource} and 143 * {@link Token#getInputStream}.</p> 144 * 145 * @param oldToken The token to copy. 146 */ 147 public this(Token oldToken) 148 { 149 type = oldToken.getType(); 150 line = oldToken.getLine(); 151 index = oldToken.getTokenIndex(); 152 charPositionInLine = oldToken.getCharPositionInLine(); 153 channel = oldToken.getChannel(); 154 start = oldToken.getStartIndex(); 155 stop = oldToken.getStopIndex(); 156 157 if (oldToken.classinfo == CommonToken.classinfo) { 158 text = (cast(CommonToken)oldToken).text; 159 source = (cast(CommonToken)oldToken).source; 160 } 161 else { 162 text = oldToken.getText(); 163 TokenFactorySourcePair sourceNew = tuple( 164 oldToken.getTokenSource, 165 oldToken.getInputStream); 166 source = sourceNew; 167 } 168 } 169 170 public int getType() 171 { 172 return type; 173 } 174 175 public void setLine(int line) 176 { 177 this.line = line; 178 } 179 180 /** 181 * @uml 182 * @override 183 */ 184 public override string getText() 185 { 186 if (text) { 187 return text; 188 } 189 190 CharStream input = getInputStream(); 191 if (input is null) return null; 192 int n = input.size(); 193 if ( start<n && stop<n) { 194 return input.getText(Interval.of(start,stop)); 195 } 196 else { 197 return "<EOF>"; 198 } 199 } 200 201 /** 202 * @uml 203 * @override 204 * Explicitly set the text for this token. If {code text} is not 205 * {@code null}, then {@link #getText} will return this value rather than 206 * extracting the text from the input. 207 * 208 * @param text The explicit text of the token, or {@code null} if the text 209 * should be obtained from the input along with the start and stop indexes 210 * of the token. 211 */ 212 public override void setText(string text) 213 { 214 this.text = text; 215 } 216 217 /** 218 * @uml 219 * @override 220 */ 221 public override int getLine() 222 { 223 return line; 224 } 225 226 /** 227 * @uml 228 * @override 229 */ 230 public override int getCharPositionInLine() 231 { 232 return charPositionInLine; 233 } 234 235 public void setCharPositionInLine(int charPositionInLine) 236 { 237 this.charPositionInLine = charPositionInLine; 238 } 239 240 /** 241 * @uml 242 * @override 243 */ 244 public override int getChannel() 245 { 246 return channel; 247 } 248 249 /** 250 * @uml 251 * @override 252 */ 253 public override void setChannel(int channel) 254 { 255 this.channel = channel; 256 } 257 258 /** 259 * @uml 260 * @override 261 */ 262 public override void setType(int type) 263 { 264 this.type = type; 265 } 266 267 /** 268 * @uml 269 * @override 270 */ 271 public override int getStartIndex() 272 { 273 return start; 274 } 275 276 public void setStartIndex(int start) 277 { 278 this.start = start; 279 } 280 281 /** 282 * @uml 283 * @override 284 */ 285 public override int getStopIndex() 286 { 287 return stop; 288 } 289 290 public void setStopIndex(int stop) 291 { 292 this.stop = stop; 293 } 294 295 /** 296 * @uml 297 * @override 298 */ 299 public override int getTokenIndex() 300 { 301 return index; 302 } 303 304 /** 305 * @uml 306 * @override 307 */ 308 public override void setTokenIndex(int index) 309 { 310 this.index = index; 311 } 312 313 /** 314 * @uml 315 * @override 316 */ 317 public override TokenSource getTokenSource() 318 { 319 return source.a; 320 } 321 322 /** 323 * @uml 324 * @override 325 */ 326 public override CharStream getInputStream() 327 { 328 return source.b; 329 } 330 331 /** 332 * @uml 333 * @override 334 */ 335 public override string toString() 336 { 337 string channelStr = ""; 338 if (channel > 0) { 339 channelStr=",channel=" ~ to!string(channel); 340 } 341 string txt = getText; 342 if (txt) { 343 txt = txt.replace("\n","\\n"); 344 txt = txt.replace("\r","\\r"); 345 txt = txt.replace("\t","\\t"); 346 } 347 else { 348 txt = "<no text>"; 349 } 350 return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(start) ~ ":" 351 ~ to!string(stop) ~ "='" ~ txt ~ "',<" ~ to!string(type) 352 ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":" 353 ~ to!string(getCharPositionInLine) ~ "]"; 354 } 355 356 }