1 module antlr.v4.runtime.ListTokenSource; 2 3 import std.conv; 4 import std..string; 5 import std.algorithm; 6 import std.typecons; 7 import antlr.v4.runtime.TokenSource; 8 import antlr.v4.runtime.Token; 9 import antlr.v4.runtime.TokenConstantDefinition; 10 import antlr.v4.runtime.TokenFactory; 11 import antlr.v4.runtime.CharStream; 12 import antlr.v4.runtime.CommonTokenFactory; 13 import antlr.v4.runtime.CommonToken; 14 15 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b"); 16 17 // Class ListTokenSource 18 /** 19 * @uml 20 * Provides an implementation of {@link TokenSource} as a wrapper around a list 21 * of {@link Token} objects. 22 * 23 * <p>If the final token in the list is an {@link Token#EOF} token, it will be used 24 * as the EOF token for every call to {@link #nextToken} after the end of the 25 * list is reached. Otherwise, an EOF token will be created.</p> 26 */ 27 class ListTokenSource : TokenSource 28 { 29 30 /** 31 * @uml 32 * The wrapped collection of {@link Token} objects to return. 33 */ 34 public Token[] tokens; 35 36 /** 37 * @uml 38 * The name of the input source. If this value is {@code null}, a call to 39 * {@link #getSourceName} should return the source name used to create the 40 * the next token in {@link #tokens} (or the previous token if the end of 41 * the input has been reached). 42 */ 43 private string sourceName; 44 45 /** 46 * @uml 47 * The index into {@link #tokens} of token to return by the next call to 48 * {@link #nextToken}. The end of the input is indicated by this value 49 * being greater than or equal to the number of items in {@link #tokens}. 50 */ 51 protected int i; 52 53 /** 54 * @uml 55 * This field caches the EOF token for the token source. 56 */ 57 protected Token eofToken; 58 59 /** 60 * @uml 61 * This is the backing field for {@link #getTokenFactory} and 62 * {@link setTokenFactory}. 63 */ 64 public TokenFactory!CommonToken _factory; 65 66 /** 67 * @uml 68 * Constructs a new {@link ListTokenSource} instance from the specified 69 * collection of {@link Token} objects. 70 * 71 * @param tokens The collection of {@link Token} objects to provide as a 72 * {@link TokenSource}. 73 * @exception NullPointerException if {@code tokens} is {@code null} 74 */ 75 public this(Token[] tokens) 76 { 77 this(tokens, null); 78 } 79 80 /** 81 * @uml 82 * Constructs a new {@link ListTokenSource} instance from the specified 83 * collection of {@link Token} objects and source name. 84 * 85 * @param tokens The collection of {@link Token} objects to provide as a 86 * {@link TokenSource}. 87 * @param sourceName The name of the {@link TokenSource}. If this value is 88 * {@code null}, {@link #getSourceName} will attempt to infer the name from 89 * the next {@link Token} (or the previous token if the end of the input has 90 * been reached). 91 * 92 * @exception NullPointerException if {@code tokens} is {@code null} 93 */ 94 public this(Token[] tokens, string sourceName) 95 in 96 { 97 assert(tokens !is null, "tokens cannot be null"); 98 } 99 do 100 { 101 _factory = CommonTokenFactory.DEFAULT; 102 this.tokens = tokens; 103 this.sourceName = sourceName; 104 } 105 106 public int getCharPositionInLine() 107 { 108 if (i < tokens.length) { 109 return tokens[i].getCharPositionInLine(); 110 } 111 else if (eofToken !is null) { 112 return eofToken.getCharPositionInLine(); 113 } 114 else if (tokens.length > 0) { 115 // have to calculate the result from the line/column of the previous 116 // token, along with the text of the token. 117 Token lastToken = tokens[$ - 1]; 118 string tokenText = lastToken.getText(); 119 if (tokenText !is null) { 120 auto lastNewLine = tokenText.lastIndexOf('\n'); 121 if (lastNewLine >= 0) { 122 return to!int((tokenText.length) - lastNewLine - 1); 123 } 124 } 125 126 return lastToken.getCharPositionInLine() + lastToken.getStopIndex() - lastToken.getStartIndex() + 1; 127 } 128 129 // only reach this if tokens is empty, meaning EOF occurs at the first 130 // position in the input 131 return 0; 132 } 133 134 public Token nextToken() 135 { 136 if (i >= tokens.length) { 137 if (eofToken is null) { 138 int start = -1; 139 if (tokens.length > 0) { 140 int previousStop = tokens[$ - 1].getStopIndex(); 141 if (previousStop != -1) { 142 start = previousStop + 1; 143 } 144 } 145 146 int stop = max(-1, start - 1); 147 TokenFactorySourcePair tfsp = tuple(this, getInputStream()); 148 eofToken = _factory.create(tfsp, TokenConstantDefinition.EOF, "EOF", TokenConstantDefinition.DEFAULT_CHANNEL, start, stop, getLine(), getCharPositionInLine()); 149 } 150 return eofToken; 151 } 152 153 Token t = tokens[i]; 154 if (i == tokens.length - 1 && t.getType() == TokenConstantDefinition.EOF) { 155 eofToken = t; 156 } 157 158 i++; 159 return t; 160 } 161 162 public int getLine() 163 { 164 if (i < tokens.length) { 165 return tokens[i].getLine(); 166 } 167 else if (eofToken !is null) { 168 return eofToken.getLine(); 169 } 170 else if (tokens.length > 0) { 171 // have to calculate the result from the line/column of the previous 172 // token, along with the text of the token. 173 Token lastToken = tokens[tokens.length - 1]; 174 int line = lastToken.getLine(); 175 176 string tokenText = lastToken.getText(); 177 if (tokenText !is null) { 178 foreach (c; tokenText) { 179 if (c == '\n') { 180 line++; 181 } 182 } 183 } 184 185 // if no text is available, assume the token did not contain any newline characters. 186 return line; 187 } 188 189 // only reach this if tokens is empty, meaning EOF occurs at the first 190 // position in the input 191 return 1; 192 } 193 194 public CharStream getInputStream() 195 { 196 if (i < tokens.length) { 197 return tokens[i].getInputStream(); 198 } 199 else if (eofToken !is null) { 200 return eofToken.getInputStream(); 201 } 202 else if (tokens.length > 0) { 203 return tokens[tokens.length - 1].getInputStream(); 204 } 205 // no input stream information is available 206 return null; 207 } 208 209 public string getSourceName() 210 { 211 if (sourceName !is null) { 212 return sourceName; 213 } 214 215 CharStream inputStream = getInputStream(); 216 if (inputStream !is null) { 217 return inputStream.getSourceName(); 218 } 219 220 return "List"; 221 } 222 223 public void tokenFactory(TokenFactory!CommonToken factory) 224 { 225 this._factory = factory; 226 } 227 228 public TokenFactory!CommonToken tokenFactory() 229 { 230 return _factory; 231 } 232 233 }