antlr.v4.runtime.CommonTokenFactory source code

1 /*
2  * Copyright (c) 2012-2019 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 module antlr.v4.runtime.CommonTokenFactory;
8 
9 import std.typecons;
10 import std.variant;
11 import std.conv;
12 import antlr.v4.runtime.TokenFactory;
13 import antlr.v4.runtime.CommonToken;
14 import antlr.v4.runtime.CharStream;
15 import antlr.v4.runtime.TokenSource;
16 import antlr.v4.runtime.misc.Interval;
17 
18 alias TokenFactorySourcePair = Tuple!(TokenSource, "a", CharStream, "b");
19 
20 /**
21  * This default implementation of {@link TokenFactory} creates
22  * {@link CommonToken} objects.
23  * <p>
24  * This token factory does not explicitly copy token text when constructing
25  * tokens.</p>
26  */
27 class CommonTokenFactory : TokenFactory!CommonToken
28 {
29 
30     /**
31      * The single instance of CommonTokenFactory.
32      * @uml
33      * @__gshared
34      */
35     private static __gshared CommonTokenFactory instance_;
36 
37     /**
38      * @uml
39      * Indicates whether {@link CommonToken#setText} should be called after
40      * constructing tokens to explicitly set the text. This is useful for cases
41      * where the input stream might not be able to provide arbitrary substrings
42      * of text from the input after the lexer creates a token (e.g. the
43      * implementation of {@link CharStream#getText} in
44      * {@link UnbufferedCharStream} throws an
45      * {@link UnsupportedOperationException}). Explicitly setting the token text
46      * allows {@link Token#getText} to be called at any time regardless of the
47      * input stream implementation.
48      *
49      * <p>
50      * The default value is {@code false} to avoid the performance and memory
51      * overhead of copying text for every token unless explicitly requested.</p>
52      */
53     protected bool copyText;
54 
55     /**
56      * @uml
57      * Constructs a {@link CommonTokenFactory} with the specified value for
58      * {@link #copyText}.
59      *
60      * <p>
61      * then {@code copyText} is {@code false}, the {@link #DEFAULT} instance
62      * should be used instead of constructing a new instance.</p>
63      *
64      *  @param copyText The value for {@link #copyText}.
65      */
66     public this(bool copyText)
67     {
68         this.copyText = copyText;
69     }
70 
71     /**
72      * @uml
73      * Constructs a {@link CommonTokenFactory} with {@link #copyText} set to
74      * {@code false}.
75      *
76      * <p>
77      * The {@link #DEFAULT} instance should be used instead of calling this
78      * directly.</p>
79      */
80     public this()
81     {
82         this(false);
83     }
84 
85     public CommonToken create(TokenFactorySourcePair source, int type, Variant text, int channel,
86         size_t start, size_t stop, int line, int charPositionInLine)
87     {
88         CommonToken t = new CommonToken(source, type, channel, start, stop);
89         t.setLine(line);
90         t.setCharPositionInLine(charPositionInLine);
91         Variant Null;
92         if (text !is Null) {
93             t.setText(text);
94         }
95         else if (copyText && source.b !is null ) {
96             Variant v = source.b.getText(Interval.of(to!int(start), to!int(stop)));
97             t.setText(v);
98         }
99         return t;
100     }
101 
102     public CommonToken create(int type, Variant text)
103     {
104         return new CommonToken(type, text);
105     }
106 
107     /**
108      * Creates the single instance of CommonTokenFactory.
109      * @uml
110      * @shared
111      */
112     private shared static this()
113     {
114         instance_ = new CommonTokenFactory;
115     }
116 
117     /**
118      * Returns: A single default instance of CommonTokenFactory.
119      */
120     public static CommonTokenFactory DEFAULT()
121     {
122         return instance_;
123     }
124 
125 }