1 /* 2 * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 module antlr.v4.runtime.ListTokenSource; 8 9 import antlr.v4.runtime.CharStream; 10 import antlr.v4.runtime.CommonToken; 11 import antlr.v4.runtime.CommonTokenFactory; 12 import antlr.v4.runtime.Token; 13 import antlr.v4.runtime.TokenConstantDefinition; 14 import antlr.v4.runtime.TokenFactory; 15 import antlr.v4.runtime.TokenSource; 16 import std.algorithm; 17 import std.conv; 18 import std.string; 19 import std.typecons; 20 import std.variant; 21 22 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 23 24 /** 25 * Provides an implementation of {@link TokenSource} as a wrapper around a list 26 * of {@link Token} objects. 27 * 28 * <p>If the final token in the list is an {@link Token#EOF} token, it will be used 29 * as the EOF token for every call to {@link #nextToken} after the end of the 30 * list is reached. Otherwise, an EOF token will be created.</p> 31 */ 32 class ListTokenSource : TokenSource 33 { 34 35 /** 36 * The wrapped collection of {@link Token} objects to return. 37 */ 38 public Token[] tokens; 39 40 /** 41 * The name of the input source. If this value is {@code null}, a call to 42 * {@link #getSourceName} should return the source name used to create the 43 * the next token in {@link #tokens} (or the previous token if the end of 44 * the input has been reached). 45 */ 46 private string sourceName; 47 48 /** 49 * The index into {@link #tokens} of token to return by the next call to 50 * {@link #nextToken}. The end of the input is indicated by this value 51 * being greater than or equal to the number of items in {@link #tokens}. 52 */ 53 protected int i; 54 55 /** 56 * This field caches the EOF token for the token source. 57 */ 58 protected Token eofToken; 59 60 /** 61 * This is the backing field for {@link #getTokenFactory} and 62 * {@link setTokenFactory}. 63 */ 64 public TokenFactory!CommonToken _factory; 65 66 /** 67 * Constructs a new {@link ListTokenSource} instance from the specified 68 * collection of {@link Token} objects. 69 * 70 * @param tokens The collection of {@link Token} objects to provide as a 71 * {@link TokenSource}. 72 * @exception NullPointerException if {@code tokens} is {@code null} 73 */ 74 public this(Token[] tokens) 75 { 76 this(tokens, null); 77 } 78 79 /** 80 * Constructs a new {@link ListTokenSource} instance from the specified 81 * collection of {@link Token} objects and source name. 82 * 83 * @param tokens The collection of {@link Token} objects to provide as a 84 * {@link TokenSource}. 85 * @param sourceName The name of the {@link TokenSource}. If this value is 86 * {@code null}, {@link #getSourceName} will attempt to infer the name from 87 * the next {@link Token} (or the previous token if the end of the input has 88 * been reached). 89 * 90 * @exception NullPointerException if {@code tokens} is {@code null} 91 */ 92 public this(Token[] tokens, string sourceName) 93 in 94 { 95 assert(tokens !is null, "tokens cannot be null"); 96 } 97 do 98 { 99 _factory = CommonTokenFactory.DEFAULT; 100 this.tokens = tokens; 101 this.sourceName = sourceName; 102 } 103 104 public int getCharPositionInLine() 105 { 106 if (i < tokens.length) { 107 return tokens[i].getCharPositionInLine(); 108 } 109 else if (eofToken !is null) { 110 return eofToken.getCharPositionInLine(); 111 } 112 else if (tokens.length > 0) { 113 // have to calculate the result from the line/column of the previous 114 // token, along with the text of the token. 115 Token lastToken = tokens[$ - 1]; 116 auto tokenText = lastToken.getText(); 117 Variant Null; 118 if (tokenText !is Null) { 119 auto lastNewLine = (to!string(tokenText)).lastIndexOf('\n'); 120 if (lastNewLine >= 0) { 121 return to!int((tokenText.length) - lastNewLine - 1); 122 } 123 } 124 125 return to!int(lastToken.getCharPositionInLine + lastToken.stopIndex - lastToken.startIndex) + 1; 126 } 127 128 // only reach this if tokens is empty, meaning EOF occurs at the first 129 // position in the input 130 return 0; 131 } 132 133 public Token nextToken() 134 { 135 if (i >= tokens.length) { 136 if (eofToken is null) { 137 size_t start = to!size_t(-1); 138 if (tokens.length > 0) { 139 auto previousStop = tokens[$ - 1].stopIndex; 140 if (previousStop != -1) { 141 start = previousStop + 1; 142 } 143 } 144 145 auto stop = max(-1, start - 1); 146 TokenFactorySourcePair tfsp = tuple(this, getInputStream()); 147 Variant v = "EOF"; 148 eofToken = _factory.create(tfsp, TokenConstantDefinition.EOF, v, TokenConstantDefinition.DEFAULT_CHANNEL, start, stop, getLine(), getCharPositionInLine()); 149 } 150 return eofToken; 151 } 152 153 Token t = tokens[i]; 154 if (i == tokens.length - 1 && t.getType() == TokenConstantDefinition.EOF) { 155 eofToken = t; 156 } 157 158 i++; 159 return t; 160 } 161 162 public int getLine() 163 { 164 if (i < tokens.length) { 165 return tokens[i].getLine(); 166 } 167 else if (eofToken !is null) { 168 return eofToken.getLine(); 169 } 170 else if (tokens.length > 0) { 171 // have to calculate the result from the line/column of the previous 172 // token, along with the text of the token. 173 Token lastToken = tokens[tokens.length - 1]; 174 int line = lastToken.getLine(); 175 176 string tokenText = to!string(lastToken.getText); 177 if (tokenText !is null) { 178 foreach (c; tokenText) { 179 if (c == '\n') { 180 line++; 181 } 182 } 183 } 184 185 // if no text is available, assume the token did not contain any newline characters. 186 return line; 187 } 188 189 // only reach this if tokens is empty, meaning EOF occurs at the first 190 // position in the input 191 return 1; 192 } 193 194 public CharStream getInputStream() 195 { 196 if (i < tokens.length) { 197 return tokens[i].getInputStream(); 198 } 199 else if (eofToken !is null) { 200 return eofToken.getInputStream(); 201 } 202 else if (tokens.length > 0) { 203 return tokens[tokens.length - 1].getInputStream(); 204 } 205 // no input stream information is available 206 return null; 207 } 208 209 public string getSourceName() 210 { 211 if (sourceName !is null) { 212 return sourceName; 213 } 214 215 CharStream inputStream = getInputStream(); 216 if (inputStream !is null) { 217 return inputStream.getSourceName(); 218 } 219 220 return "List"; 221 } 222 223 public void tokenFactory(TokenFactory!CommonToken factory) 224 { 225 this._factory = factory; 226 } 227 228 public TokenFactory!CommonToken tokenFactory() 229 { 230 return _factory; 231 } 232 233 }