antlr.v4.runtime.CommonTokenFactory source code

1 /*
2  * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonTokenFactory;
8 
9 import std.typecons;
10 import std.variant;
11 import antlr.v4.runtime.TokenFactory;
12 import antlr.v4.runtime.CommonToken;
13 import antlr.v4.runtime.CharStream;
14 import antlr.v4.runtime.TokenSource;
15 import antlr.v4.runtime.misc.Interval;
16 
17 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
18 
19 /**
20  * This default implementation of {@link TokenFactory} creates
21  * {@link CommonToken} objects.
22  * <p>
23  * This token factory does not explicitly copy token text when constructing
24  * tokens.</p>
25  */
26 class CommonTokenFactory : TokenFactory!CommonToken
27 {
28 
29     /**
30      * The single instance of CommonTokenFactory.
31      * @uml
32      * @__gshared
33      */
34     private static __gshared CommonTokenFactory instance_;
35 
36     /**
37      * @uml
38      * Indicates whether {@link CommonToken#setText} should be called after
39      * constructing tokens to explicitly set the text. This is useful for cases
40      * where the input stream might not be able to provide arbitrary substrings
41      * of text from the input after the lexer creates a token (e.g. the
42      * implementation of {@link CharStream#getText} in
43      * {@link UnbufferedCharStream} throws an
44      * {@link UnsupportedOperationException}). Explicitly setting the token text
45      * allows {@link Token#getText} to be called at any time regardless of the
46      * input stream implementation.
47      *
48      * <p>
49      * The default value is {@code false} to avoid the performance and memory
50      * overhead of copying text for every token unless explicitly requested.</p>
51      */
52     protected bool copyText;
53 
54     /**
55      * @uml
56      * Constructs a {@link CommonTokenFactory} with the specified value for
57      * {@link #copyText}.
58      *
59      * <p>
60      * then {@code copyText} is {@code false}, the {@link #DEFAULT} instance
61      * should be used instead of constructing a new instance.</p>
62      *
63      *  @param copyText The value for {@link #copyText}.
64      */
65     public this(bool copyText)
66     {
67         this.copyText = copyText;
68     }
69 
70     /**
71      * @uml
72      * Constructs a {@link CommonTokenFactory} with {@link #copyText} set to
73      * {@code false}.
74      *
75      * <p>
76      * The {@link #DEFAULT} instance should be used instead of calling this
77      * directly.</p>
78      */
79     public this()
80     {
81         this(false);
82     }
83 
84     public CommonToken create(TokenFactorySourcePair source, int type, Variant text, int channel,
85         int start, int stop, int line, int charPositionInLine)
86     {
87         CommonToken t = new CommonToken(source, type, channel, start, stop);
88         t.setLine(line);
89         t.setCharPositionInLine(charPositionInLine);
90         Variant Null;
91         if (text !is Null) {
92             t.setText(text);
93         }
94         else if (copyText && source.b !is null ) {
95             Variant v = source.b.getText(Interval.of(start,stop));
96             t.setText(v);
97         }
98         return t;
99     }
100 
101     public CommonToken create(int type, Variant text)
102     {
103         return new CommonToken(type, text);
104     }
105 
106     /**
107      * Creates the single instance of CommonTokenFactory.
108      * @uml
109      * @shared
110      */
111     private shared static this()
112     {
113         instance_ = new CommonTokenFactory;
114     }
115 
116     /**
117      * Returns: A single default instance of CommonTokenFactory.
118      */
119     public static CommonTokenFactory DEFAULT()
120     {
121         return instance_;
122     }
123 
124 }