1 /*
2  * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.ListTokenSource;
8 
9 import antlr.v4.runtime.CharStream;
10 import antlr.v4.runtime.CommonToken;
11 import antlr.v4.runtime.CommonTokenFactory;
12 import antlr.v4.runtime.Token;
13 import antlr.v4.runtime.TokenConstantDefinition;
14 import antlr.v4.runtime.TokenFactory;
15 import antlr.v4.runtime.TokenSource;
16 import std.algorithm;
17 import std.conv;
18 import std.string;
19 import std.typecons;
20 import std.variant;
21 
22 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
23 
24 /**
25  * Provides an implementation of {@link TokenSource} as a wrapper around a list
26  * of {@link Token} objects.
27  *
28  * <p>If the final token in the list is an {@link Token#EOF} token, it will be used
29  * as the EOF token for every call to {@link #nextToken} after the end of the
30  * list is reached. Otherwise, an EOF token will be created.</p>
31  */
32 class ListTokenSource : TokenSource
33 {
34 
35     /**
36      * The wrapped collection of {@link Token} objects to return.
37      */
38     public Token[] tokens;
39 
40     /**
41      * The name of the input source. If this value is {@code null}, a call to
42      * {@link #getSourceName} should return the source name used to create the
43      * the next token in {@link #tokens} (or the previous token if the end of
44      * the input has been reached).
45      */
46     private string sourceName;
47 
48     /**
49      * The index into {@link #tokens} of token to return by the next call to
50      * {@link #nextToken}. The end of the input is indicated by this value
51      * being greater than or equal to the number of items in {@link #tokens}.
52      */
53     protected int i;
54 
55     /**
56      * This field caches the EOF token for the token source.
57      */
58     protected Token eofToken;
59 
60     /**
61      * This is the backing field for {@link #getTokenFactory} and
62      *  {@link setTokenFactory}.
63      */
64     public TokenFactory!CommonToken _factory;
65 
66     /**
67      * Constructs a new {@link ListTokenSource} instance from the specified
68      * collection of {@link Token} objects.
69      *
70      *  @param tokens The collection of {@link Token} objects to provide as a
71      *  {@link TokenSource}.
72      *  @exception NullPointerException if {@code tokens} is {@code null}
73      */
74     public this(Token[] tokens)
75     {
76         this(tokens, null);
77     }
78 
79     /**
80      * Constructs a new {@link ListTokenSource} instance from the specified
81      * collection of {@link Token} objects and source name.
82      *
83      *  @param tokens The collection of {@link Token} objects to provide as a
84      * {@link TokenSource}.
85      *  @param sourceName The name of the {@link TokenSource}. If this value is
86      * {@code null}, {@link #getSourceName} will attempt to infer the name from
87      * the next {@link Token} (or the previous token if the end of the input has
88      * been reached).
89      *
90      *  @exception NullPointerException if {@code tokens} is {@code null}
91      */
92     public this(Token[] tokens, string sourceName)
93     in
94     {
95         assert(tokens !is null, "tokens cannot be null");
96     }
97     do
98     {
99         _factory = CommonTokenFactory.DEFAULT;
100         this.tokens = tokens;
101         this.sourceName = sourceName;
102     }
103 
104     public int getCharPositionInLine()
105     {
106     if (i < tokens.length) {
107             return tokens[i].getCharPositionInLine();
108         }
109         else if (eofToken !is null) {
110             return eofToken.getCharPositionInLine();
111         }
112         else if (tokens.length > 0) {
113             // have to calculate the result from the line/column of the previous
114             // token, along with the text of the token.
115             Token lastToken = tokens[$ - 1];
116             auto tokenText = lastToken.getText();
117                         Variant Null;
118             if (tokenText !is Null) {
119                             auto lastNewLine = (to!string(tokenText)).lastIndexOf('\n');
120                 if (lastNewLine >= 0) {
121                                     return to!int((tokenText.length) - lastNewLine - 1);
122                 }
123             }
124 
125             return to!int(lastToken.getCharPositionInLine + lastToken.stopIndex - lastToken.startIndex) + 1;
126         }
127 
128         // only reach this if tokens is empty, meaning EOF occurs at the first
129         // position in the input
130         return 0;
131     }
132 
133     public Token nextToken()
134     {
135     if (i >= tokens.length) {
136             if (eofToken is null) {
137                 size_t start = to!size_t(-1);
138                 if (tokens.length > 0) {
139                     auto previousStop = tokens[$ - 1].stopIndex;
140                     if (previousStop != -1) {
141                         start = previousStop + 1;
142                     }
143                 }
144 
145                 auto stop = max(-1, start - 1);
146                 TokenFactorySourcePair tfsp = tuple(this, getInputStream());
147                 Variant v = "EOF";
148                 eofToken = _factory.create(tfsp, TokenConstantDefinition.EOF, v, TokenConstantDefinition.DEFAULT_CHANNEL, start, stop, getLine(), getCharPositionInLine());
149             }
150             return eofToken;
151         }
152 
153         Token t = tokens[i];
154         if (i == tokens.length - 1 && t.getType() == TokenConstantDefinition.EOF) {
155             eofToken = t;
156         }
157 
158         i++;
159         return t;
160     }
161 
162     public int getLine()
163     {
164     if (i < tokens.length) {
165             return tokens[i].getLine();
166         }
167         else if (eofToken !is null) {
168             return eofToken.getLine();
169         }
170         else if (tokens.length > 0) {
171             // have to calculate the result from the line/column of the previous
172             // token, along with the text of the token.
173             Token lastToken = tokens[tokens.length - 1];
174             int line = lastToken.getLine();
175 
176             string tokenText = to!string(lastToken.getText);
177             if (tokenText !is null) {
178                 foreach (c; tokenText) {
179                     if (c == '\n') {
180                         line++;
181                     }
182                 }
183             }
184 
185             // if no text is available, assume the token did not contain any newline characters.
186             return line;
187         }
188 
189         // only reach this if tokens is empty, meaning EOF occurs at the first
190         // position in the input
191         return 1;
192     }
193 
194     public CharStream getInputStream()
195     {
196     if (i < tokens.length) {
197             return tokens[i].getInputStream();
198         }
199         else if (eofToken !is null) {
200             return eofToken.getInputStream();
201         }
202         else if (tokens.length > 0) {
203             return tokens[tokens.length - 1].getInputStream();
204         }
205         // no input stream information is available
206         return null;
207     }
208 
209     public string getSourceName()
210     {
211         if (sourceName !is null) {
212             return sourceName;
213         }
214 
215         CharStream inputStream = getInputStream();
216         if (inputStream !is null) {
217             return inputStream.getSourceName();
218         }
219 
220         return "List";
221     }
222 
223     public void tokenFactory(TokenFactory!CommonToken factory)
224     {
225         this._factory = factory;
226     }
227 
228     public TokenFactory!CommonToken tokenFactory()
229     {
230         return _factory;
231     }
232 
233 }