public class Tokenizer
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
private static int[] |
DELIMITERS |
private java.lang.String |
input |
private static int |
NON |
private static int |
SEQUENCE_TOKEN |
private static int |
SINGLE_TOKEN |
private static boolean[] |
STRING_TERMINATORS |
Constructor and Description |
---|
Tokenizer(java.lang.String input) |
Modifier and Type | Method and Description |
---|---|
private static void |
bakeToken(java.util.List<Token> tokens,
char[] input,
int i,
int start) |
static java.lang.String |
detokenize(java.util.List<Token> tokens) |
private static boolean |
isDelimiter(char c) |
(package private) static boolean |
isSingleTokenDelimiter(char c) |
private static boolean |
isWhitespace(char c) |
java.util.List<Token> |
tokenize() |
private final java.lang.String input
private static final int NON
private static final int SINGLE_TOKEN
private static final int SEQUENCE_TOKEN
private static final int[] DELIMITERS
private static final boolean[] STRING_TERMINATORS
public java.util.List<Token> tokenize()
private static boolean isWhitespace(char c)
static boolean isSingleTokenDelimiter(char c)
public static java.lang.String detokenize(java.util.List<Token> tokens)
private static boolean isDelimiter(char c)
private static void bakeToken(java.util.List<Token> tokens, char[] input, int i, int start)