1 /* 2 * Copyright (c) 2012-2018 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 module antlr.v4.runtime.CommonToken; 8 9 import std.conv; 10 import std.array; 11 import std.typecons; 12 import std.container : DList; 13 import antlr.v4.runtime.WritableToken; 14 import antlr.v4.runtime.CharStream; 15 import antlr.v4.runtime.Token; 16 import antlr.v4.runtime.TokenConstantDefinition; 17 import antlr.v4.runtime.TokenSource; 18 import antlr.v4.runtime.misc.Interval; 19 20 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 21 22 /** 23 * TODO add class description 24 */ 25 class CommonToken : WritableToken 26 { 27 28 /** 29 * An empty {@link Pair} which is used as the default value of 30 * {@link #source} for tokens that do not have a source. 31 */ 32 protected static TokenFactorySourcePair EMPTY_SOURCE; 33 34 /** 35 * This is the backing field for {@link #getType} and {@link #setType}. 36 */ 37 protected int type; 38 39 /** 40 * This is the backing field for {@link #getLine} and {@link #setLine}. 41 */ 42 protected int line; 43 44 /** 45 * This is the backing field for {@link #getCharPositionInLine} and 46 * {@link #setCharPositionInLine}. 47 */ 48 protected int charPositionInLine = -1; 49 50 /** 51 * This is the backing field for {@link #getChannel} and 52 * {@link #setChannel}. 53 */ 54 protected int channel = TokenConstantDefinition.DEFAULT_CHANNEL; 55 56 /** 57 * This is the backing field for {@link #getTokenSource} and 58 * {@link #getInputStream}. 59 * 60 * <p> 61 * These properties share a field to reduce the memory footprint of 62 * {@link CommonToken}. Tokens created by a {@link CommonTokenFactory} from 63 * the same source and input stream share a reference to the same 64 * {@link Pair} containing these values.</p> 65 */ 66 protected TokenFactorySourcePair source; 67 68 /** 69 * This is the backing field for {@link #getText} when the token text is 70 * explicitly set in the constructor or via {@link #setText}. 71 * 72 * @see #getText() 73 */ 74 protected string text; 75 76 /** 77 * This is the backing field for {@link #getTokenIndex} and 78 * {@link #setTokenIndex}. 79 */ 80 protected int index = -1; 81 82 /** 83 * This is the backing field for {@link #getStartIndex} and 84 * {@link #setStartIndex}. 85 */ 86 protected int start; 87 88 /** 89 * This is the backing field for {@link #getStopIndex} and 90 * {@link #setStopIndex}. 91 */ 92 protected int stop; 93 94 /** 95 * Constructs a new {@link CommonToken} with the specified token type. 96 * 97 * @param type The token type. 98 */ 99 public this(int type) 100 { 101 this.type = type; 102 this.source = EMPTY_SOURCE; 103 } 104 105 public this(TokenFactorySourcePair source, int type, int channel, int start, int stop) 106 { 107 this.source = source; 108 this.type = type; 109 this.channel = channel; 110 this.start = start; 111 this.stop = stop; 112 if (source.a !is null) { 113 this.line = source.a.getLine(); 114 this.charPositionInLine = source.a.getCharPositionInLine(); 115 } 116 } 117 118 /** 119 * Constructs a new {@link CommonToken} with the specified token type and 120 * text. 121 * 122 * @param type The token type. 123 * @param text The text of the token. 124 */ 125 public this(int type, string text) 126 { 127 this.type = type; 128 this.channel = TokenConstantDefinition.DEFAULT_CHANNEL; 129 this.text = text; 130 this.source = EMPTY_SOURCE; 131 } 132 133 /** 134 * Constructs a new {@link CommonToken} as a copy of another {@link Token}. 135 * * 136 * <p> 137 * If {@code oldToken} is also a {@link CommonToken} instance, the newly 138 * constructed token will share a reference to the {@link #text} field and 139 * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will 140 * be assigned the result of calling {@link #getText}, and {@link #source} 141 * will be constructed from the result of {@link Token#getTokenSource} and 142 * {@link Token#getInputStream}.</p> 143 * 144 * @param oldToken The token to copy. 145 */ 146 public this(Token oldToken) 147 { 148 type = oldToken.getType(); 149 line = oldToken.getLine(); 150 index = oldToken.getTokenIndex(); 151 charPositionInLine = oldToken.getCharPositionInLine(); 152 channel = oldToken.getChannel(); 153 start = oldToken.getStartIndex(); 154 stop = oldToken.getStopIndex(); 155 156 if (oldToken.classinfo == CommonToken.classinfo) { 157 text = (cast(CommonToken)oldToken).text; 158 source = (cast(CommonToken)oldToken).source; 159 } 160 else { 161 text = oldToken.getText(); 162 TokenFactorySourcePair sourceNew = tuple( 163 oldToken.getTokenSource, 164 oldToken.getInputStream); 165 source = sourceNew; 166 } 167 } 168 169 public int getType() 170 { 171 return type; 172 } 173 174 public void setLine(int line) 175 { 176 this.line = line; 177 } 178 179 /** 180 * @uml 181 * @override 182 */ 183 public override string getText() 184 { 185 if (text) { 186 return text; 187 } 188 189 CharStream input = getInputStream(); 190 if (input is null) return null; 191 int n = input.size(); 192 if ( start<n && stop<n) { 193 return input.getText(Interval.of(start,stop)); 194 } 195 else { 196 return "<EOF>"; 197 } 198 } 199 200 /** 201 * @uml 202 * @override 203 * Explicitly set the text for this token. If {code text} is not 204 * {@code null}, then {@link #getText} will return this value rather than 205 * extracting the text from the input. 206 * 207 * @param text The explicit text of the token, or {@code null} if the text 208 * should be obtained from the input along with the start and stop indexes 209 * of the token. 210 */ 211 public override void setText(string text) 212 { 213 this.text = text; 214 } 215 216 /** 217 * @uml 218 * @override 219 */ 220 public override int getLine() 221 { 222 return line; 223 } 224 225 /** 226 * @uml 227 * @override 228 */ 229 public override int getCharPositionInLine() 230 { 231 return charPositionInLine; 232 } 233 234 public void setCharPositionInLine(int charPositionInLine) 235 { 236 this.charPositionInLine = charPositionInLine; 237 } 238 239 /** 240 * @uml 241 * @override 242 */ 243 public override int getChannel() 244 { 245 return channel; 246 } 247 248 /** 249 * @uml 250 * @override 251 */ 252 public override void setChannel(int channel) 253 { 254 this.channel = channel; 255 } 256 257 /** 258 * @uml 259 * @override 260 */ 261 public override void setType(int type) 262 { 263 this.type = type; 264 } 265 266 /** 267 * @uml 268 * @override 269 */ 270 public override int getStartIndex() 271 { 272 return start; 273 } 274 275 public void setStartIndex(int start) 276 { 277 this.start = start; 278 } 279 280 /** 281 * @uml 282 * @override 283 */ 284 public override int getStopIndex() 285 { 286 return stop; 287 } 288 289 public void setStopIndex(int stop) 290 { 291 this.stop = stop; 292 } 293 294 /** 295 * @uml 296 * @override 297 */ 298 public override int getTokenIndex() 299 { 300 return index; 301 } 302 303 /** 304 * @uml 305 * @override 306 */ 307 public override void setTokenIndex(int index) 308 { 309 this.index = index; 310 } 311 312 /** 313 * @uml 314 * @override 315 */ 316 public override TokenSource getTokenSource() 317 { 318 return source.a; 319 } 320 321 /** 322 * @uml 323 * @override 324 */ 325 public override CharStream getInputStream() 326 { 327 return source.b; 328 } 329 330 /** 331 * @uml 332 * @override 333 */ 334 public override string toString() 335 { 336 string channelStr = ""; 337 if (channel > 0) { 338 channelStr=",channel=" ~ to!string(channel); 339 } 340 string txt = getText; 341 if (txt) { 342 txt = txt.replace("\n","\\n"); 343 txt = txt.replace("\r","\\r"); 344 txt = txt.replace("\t","\\t"); 345 } 346 else { 347 txt = "<no text>"; 348 } 349 return "[@" ~ to!string(getTokenIndex) ~ "," ~ to!string(start) ~ ":" 350 ~ to!string(stop) ~ "='" ~ txt ~ "',<" ~ to!string(type) 351 ~ ">" ~ channelStr ~ "," ~ to!string(line) ~ ":" 352 ~ to!string(getCharPositionInLine) ~ "]"; 353 } 354 355 }