1 module antlr.v4.runtime.ListTokenSource; 2 3 import std.conv; 4 import std.string; 5 import std.algorithm; 6 import std.typecons; 7 import antlr.v4.runtime.TokenSource; 8 import antlr.v4.runtime.Token; 9 import antlr.v4.runtime.TokenConstantDefinition; 10 import antlr.v4.runtime.TokenFactory; 11 import antlr.v4.runtime.CharStream; 12 import antlr.v4.runtime.CommonTokenFactory; 13 import antlr.v4.runtime.CommonToken; 14 15 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 16 17 /** 18 * @uml 19 * Provides an implementation of {@link TokenSource} as a wrapper around a list 20 * of {@link Token} objects. 21 * 22 * <p>If the final token in the list is an {@link Token#EOF} token, it will be used 23 * as the EOF token for every call to {@link #nextToken} after the end of the 24 * list is reached. Otherwise, an EOF token will be created.</p> 25 */ 26 class ListTokenSource : TokenSource 27 { 28 29 /** 30 * @uml 31 * The wrapped collection of {@link Token} objects to return. 32 */ 33 public Token[] tokens; 34 35 /** 36 * @uml 37 * The name of the input source. If this value is {@code null}, a call to 38 * {@link #getSourceName} should return the source name used to create the 39 * the next token in {@link #tokens} (or the previous token if the end of 40 * the input has been reached). 41 */ 42 private string sourceName; 43 44 /** 45 * @uml 46 * The index into {@link #tokens} of token to return by the next call to 47 * {@link #nextToken}. The end of the input is indicated by this value 48 * being greater than or equal to the number of items in {@link #tokens}. 49 */ 50 protected int i; 51 52 /** 53 * @uml 54 * This field caches the EOF token for the token source. 55 */ 56 protected Token eofToken; 57 58 /** 59 * @uml 60 * This is the backing field for {@link #getTokenFactory} and 61 * {@link setTokenFactory}. 62 */ 63 public TokenFactory!CommonToken _factory; 64 65 /** 66 * @uml 67 * Constructs a new {@link ListTokenSource} instance from the specified 68 * collection of {@link Token} objects. 69 * 70 * @param tokens The collection of {@link Token} objects to provide as a 71 * {@link TokenSource}. 72 * @exception NullPointerException if {@code tokens} is {@code null} 73 */ 74 public this(Token[] tokens) 75 { 76 this(tokens, null); 77 } 78 79 /** 80 * @uml 81 * Constructs a new {@link ListTokenSource} instance from the specified 82 * collection of {@link Token} objects and source name. 83 * 84 * @param tokens The collection of {@link Token} objects to provide as a 85 * {@link TokenSource}. 86 * @param sourceName The name of the {@link TokenSource}. If this value is 87 * {@code null}, {@link #getSourceName} will attempt to infer the name from 88 * the next {@link Token} (or the previous token if the end of the input has 89 * been reached). 90 * 91 * @exception NullPointerException if {@code tokens} is {@code null} 92 */ 93 public this(Token[] tokens, string sourceName) 94 in 95 { 96 assert(tokens !is null, "tokens cannot be null"); 97 } 98 do 99 { 100 _factory = CommonTokenFactory.DEFAULT; 101 this.tokens = tokens; 102 this.sourceName = sourceName; 103 } 104 105 public int getCharPositionInLine() 106 { 107 if (i < tokens.length) { 108 return tokens[i].getCharPositionInLine(); 109 } 110 else if (eofToken !is null) { 111 return eofToken.getCharPositionInLine(); 112 } 113 else if (tokens.length > 0) { 114 // have to calculate the result from the line/column of the previous 115 // token, along with the text of the token. 116 Token lastToken = tokens[$ - 1]; 117 string tokenText = lastToken.getText(); 118 if (tokenText !is null) { 119 auto lastNewLine = tokenText.lastIndexOf('\n'); 120 if (lastNewLine >= 0) { 121 return to!int((tokenText.length) - lastNewLine - 1); 122 } 123 } 124 125 return lastToken.getCharPositionInLine() + lastToken.getStopIndex() - lastToken.getStartIndex() + 1; 126 } 127 128 // only reach this if tokens is empty, meaning EOF occurs at the first 129 // position in the input 130 return 0; 131 } 132 133 public Token nextToken() 134 { 135 if (i >= tokens.length) { 136 if (eofToken is null) { 137 int start = -1; 138 if (tokens.length > 0) { 139 int previousStop = tokens[$ - 1].getStopIndex(); 140 if (previousStop != -1) { 141 start = previousStop + 1; 142 } 143 } 144 145 int stop = max(-1, start - 1); 146 TokenFactorySourcePair tfsp = tuple(this, getInputStream()); 147 eofToken = _factory.create(tfsp, TokenConstantDefinition.EOF, "EOF", TokenConstantDefinition.DEFAULT_CHANNEL, start, stop, getLine(), getCharPositionInLine()); 148 } 149 return eofToken; 150 } 151 152 Token t = tokens[i]; 153 if (i == tokens.length - 1 && t.getType() == TokenConstantDefinition.EOF) { 154 eofToken = t; 155 } 156 157 i++; 158 return t; 159 } 160 161 public int getLine() 162 { 163 if (i < tokens.length) { 164 return tokens[i].getLine(); 165 } 166 else if (eofToken !is null) { 167 return eofToken.getLine(); 168 } 169 else if (tokens.length > 0) { 170 // have to calculate the result from the line/column of the previous 171 // token, along with the text of the token. 172 Token lastToken = tokens[tokens.length - 1]; 173 int line = lastToken.getLine(); 174 175 string tokenText = lastToken.getText(); 176 if (tokenText !is null) { 177 foreach (c; tokenText) { 178 if (c == '\n') { 179 line++; 180 } 181 } 182 } 183 184 // if no text is available, assume the token did not contain any newline characters. 185 return line; 186 } 187 188 // only reach this if tokens is empty, meaning EOF occurs at the first 189 // position in the input 190 return 1; 191 } 192 193 public CharStream getInputStream() 194 { 195 if (i < tokens.length) { 196 return tokens[i].getInputStream(); 197 } 198 else if (eofToken !is null) { 199 return eofToken.getInputStream(); 200 } 201 else if (tokens.length > 0) { 202 return tokens[tokens.length - 1].getInputStream(); 203 } 204 // no input stream information is available 205 return null; 206 } 207 208 public string getSourceName() 209 { 210 if (sourceName !is null) { 211 return sourceName; 212 } 213 214 CharStream inputStream = getInputStream(); 215 if (inputStream !is null) { 216 return inputStream.getSourceName(); 217 } 218 219 return "List"; 220 } 221 222 public void tokenFactory(TokenFactory!CommonToken factory) 223 { 224 this._factory = factory; 225 } 226 227 public TokenFactory!CommonToken tokenFactory() 228 { 229 return _factory; 230 } 231 232 }