1 module antlr.v4.runtime.UnbufferedTokenStream; 2 3 import std.format; 4 import std.array; 5 import std.conv; 6 import std.algorithm; 7 import antlr.v4.runtime.RuleContext; 8 import antlr.v4.runtime.TokenStream; 9 import antlr.v4.runtime.Token; 10 import antlr.v4.runtime.TokenSource; 11 import antlr.v4.runtime.TokenConstantDefinition; 12 import antlr.v4.runtime.UnsupportedOperationException; 13 import antlr.v4.runtime.IllegalArgumentException; 14 import antlr.v4.runtime.IllegalStateException; 15 import antlr.v4.runtime.WritableToken; 16 import antlr.v4.runtime.misc.Interval; 17 18 // Class UnbufferedTokenStream 19 /** 20 * TODO add class description 21 */ 22 class UnbufferedTokenStream : TokenStream 23 { 24 25 protected TokenSource tokenSource; 26 27 /** 28 * @uml 29 * A moving window buffer of the data being scanned. While there's a marker, 30 * we keep adding to buffer. Otherwise, {@link #consume consume()} resets so 31 * we start filling at index 0 again. 32 */ 33 protected Token[] tokens; 34 35 /** 36 * @uml 37 * The number of tokens currently in {@link #tokens tokens}. 38 * 39 * <p>This is not the buffer capacity, that's {@code tokens.length}.</p> 40 */ 41 protected int n; 42 43 /** 44 * @uml 45 * 0..n-1 index into {@link #tokens tokens} of next token. 46 * 47 * <p>The {@code LT(1)} token is {@code tokens[p]}. If {@code p == n}, we are 48 * out of buffered tokens.</p> 49 */ 50 protected int p = 0; 51 52 /** 53 * @uml 54 * Count up with {@link #mark mark()} and down with 55 * {@link #release release()}. When we {@code release()} the last mark, 56 * {@code numMarkers} reaches 0 and we reset the buffer. Copy 57 * {@code tokens[p]..tokens[n-1]} to {@code tokens[0]..tokens[(n-1)-p]}. 58 */ 59 protected int numMarkers = 0; 60 61 protected Token lastToken; 62 63 protected Token lastTokenBufferStart; 64 65 protected int currentTokenIndex = 0; 66 67 public this(TokenSource tokenSource) 68 { 69 this(tokenSource, 256); 70 } 71 72 public this(TokenSource tokenSource, int bufferSize) 73 { 74 this.tokenSource = tokenSource; 75 tokens = new Token[bufferSize]; 76 n = 0; 77 fill(1); // prime the pump 78 } 79 80 public Token get(int i) 81 { 82 int bufferStartIndex = getBufferStartIndex(); 83 assert(i < bufferStartIndex || i >= bufferStartIndex + n, 84 format("get(%1$s) outside buffer: %2$s..%3$s", i, bufferStartIndex, 85 bufferStartIndex+n)); 86 return tokens[i - bufferStartIndex]; 87 } 88 89 public Token LT(int i) 90 { 91 if ( i==-1 ) { 92 return lastToken; 93 } 94 95 sync(i); 96 int index = p + i - 1; 97 assert(index >= 0, format("LT(%s) gives negative index",i)); 98 if ( index >= n ) { 99 assert (n > 0 && tokens[n-1].getType() == TokenConstantDefinition.EOF); 100 return tokens[n-1]; 101 } 102 return tokens[index]; 103 } 104 105 public int LA(int i) 106 { 107 return LT(i).getType(); 108 } 109 110 public TokenSource getTokenSource() 111 { 112 return tokenSource; 113 } 114 115 public string getText() 116 { 117 return ""; 118 } 119 120 public string getText(RuleContext ctx) 121 { 122 return getText(ctx.getSourceInterval()); 123 } 124 125 public string getText(Token start, Token stop) 126 { 127 return getText(Interval.of(start.getTokenIndex(), stop.getTokenIndex())); 128 } 129 130 public void consume() 131 { 132 if (LA(1) == TokenConstantDefinition.EOF) { 133 throw new IllegalStateException("cannot consume EOF"); 134 } 135 // buf always has at least tokens[p==0] in this method due to ctor 136 lastToken = tokens[p]; // track last token for LT(-1) 137 // if we're at last token and no markers, opportunity to flush buffer 138 if ( p == n-1 && numMarkers==0 ) { 139 n = 0; 140 p = -1; // p++ will leave this at 0 141 lastTokenBufferStart = lastToken; 142 } 143 p++; 144 currentTokenIndex++; 145 sync(1); 146 } 147 148 /** 149 * Make sure we have 'need' elements from current position {@link #p p}. Last valid 150 * {@code p} index is {@code tokens.length-1}. {@code p+need-1} is the tokens index 'need' elements 151 * ahead. If we need 1 element, {@code (p+1-1)==p} must be less than {@code tokens.length}. 152 */ 153 protected void sync(int want) 154 { 155 int need = (p+want-1) - n + 1; // how many more elements we need? 156 if ( need > 0 ) { 157 fill(need); 158 } 159 } 160 161 /** 162 * Add {@code n} elements to the buffer. Returns the number of tokens 163 * actually added to the buffer. If the return value is less than {@code n}, 164 * then EOF was reached before {@code n} tokens could be added. 165 */ 166 protected int fill(int n) 167 { 168 for (int i=0; i<n; i++) { 169 if (this.n > 0 && tokens[this.n-1].getType() == TokenConstantDefinition.EOF) { 170 return i; 171 } 172 Token t = tokenSource.nextToken(); 173 add(t); 174 } 175 return n; 176 } 177 178 protected void add(Token t) 179 { 180 if (n >= tokens.length) { 181 tokens.length = tokens.length * 2; 182 } 183 if (cast(WritableToken)t) { 184 (cast(WritableToken)t).setTokenIndex(getBufferStartIndex() + n); 185 } 186 tokens[n++] = t; 187 } 188 189 /** 190 * Return a marker that we can release later. 191 * 192 * <p>The specific marker value used for this class allows for some level of 193 * protection against misuse where {@code seek()} is called on a mark or 194 * {@code release()} is called in the wrong order.</p> 195 */ 196 public int mark() 197 { 198 if (numMarkers == 0) { 199 lastTokenBufferStart = lastToken; 200 } 201 int mark = -numMarkers - 1; 202 numMarkers++; 203 return mark; 204 } 205 206 public void release(int marker) 207 { 208 int expectedMark = -numMarkers; 209 if ( marker!=expectedMark ) { 210 throw new IllegalStateException("release() called with an invalid marker."); 211 } 212 213 numMarkers--; 214 if (numMarkers == 0) { // can we release buffer? 215 if (p > 0) { 216 // Copy tokens[p]..tokens[n-1] to tokens[0]..tokens[(n-1)-p], reset ptrs 217 // p is last valid token; move nothing if p==n as we have no valid char 218 // System.arraycopy(tokens, p, tokens, 0, n - p); // shift n-p tokens from p to 0 219 tokens = tokens[p..(n-p)]; 220 n = n - p; 221 p = 0; 222 } 223 lastTokenBufferStart = lastToken; 224 } 225 } 226 227 /** 228 * @uml 229 * @override 230 */ 231 public override int index() 232 { 233 return currentTokenIndex; 234 } 235 236 public void seek(int index) 237 { 238 if (index == currentTokenIndex) { 239 return; 240 } 241 242 if (index > currentTokenIndex) { 243 sync(index - currentTokenIndex); 244 index = min(index, getBufferStartIndex() + n - 1); 245 } 246 247 int bufferStartIndex = getBufferStartIndex(); 248 int i = index - bufferStartIndex; 249 if ( i < 0 ) { 250 throw new IllegalArgumentException("cannot seek to negative index " ~ 251 to!string(index)); 252 } 253 else if (i >= n) { 254 auto msg = format("seek to index outside buffer: %1$s not in %2$s..%3$s", 255 index, bufferStartIndex, bufferStartIndex + n); 256 throw new UnsupportedOperationException(msg); 257 } 258 259 p = i; 260 currentTokenIndex = index; 261 if (p == 0) { 262 lastToken = lastTokenBufferStart; 263 } 264 else { 265 lastToken = tokens[p-1]; 266 } 267 268 } 269 270 public int size() 271 { 272 throw new UnsupportedOperationException("Unbuffered stream cannot know its size"); 273 } 274 275 public string getSourceName() 276 { 277 return tokenSource.getSourceName(); 278 } 279 280 public string getText(Interval interval) 281 { 282 int bufferStartIndex = getBufferStartIndex(); 283 int bufferStopIndex = bufferStartIndex + to!int(tokens.length) - 1; 284 285 int start = interval.a; 286 int stop = interval.b; 287 if (start < bufferStartIndex || stop > bufferStopIndex) { 288 auto msg = format("interval %1$s not in token buffer window: %2$s..%3$s", 289 interval, bufferStartIndex, bufferStopIndex); 290 throw new UnsupportedOperationException(msg); 291 } 292 293 int a = start - bufferStartIndex; 294 int b = stop - bufferStartIndex; 295 296 auto buf = appender!string; 297 for (int i = a; i <= b; i++) { 298 Token t = tokens[i]; 299 buf.put(t.getText); 300 } 301 302 return buf.data; 303 } 304 305 protected int getBufferStartIndex() 306 { 307 return currentTokenIndex - p; 308 } 309 310 }