7#ifndef STANDARDTOKENIZERIMPL_H
8#define STANDARDTOKENIZERIMPL_H
#define LUCENE_CLASS(Name)
Definition LuceneObject.h:24
Base class for all Lucene classes.
Definition LuceneObject.h:31
Definition StandardTokenizerImpl.h:14
int32_t yycolumn
The number of characters from the last newline up to the start of the matched text.
Definition StandardTokenizerImpl.h:119
int32_t zzEndRead
EndRead marks the last character in the buffer, that has been read from input.
Definition StandardTokenizerImpl.h:110
int32_t zzMarkedPos
The text position at the last accepting state.
Definition StandardTokenizerImpl.h:98
int32_t zzCurrentPos
The current text position in the buffer.
Definition StandardTokenizerImpl.h:104
void yyclose()
Closes the input stream.
static const int32_t ZZ_ACTION_LENGTH
Definition StandardTokenizerImpl.h:41
int32_t zzLexicalState
The current lexical state.
Definition StandardTokenizerImpl.h:92
static IntArray _ZZ_ROWMAP
Translates a state to a row index in the transition table.
Definition StandardTokenizerImpl.h:49
static void ZZ_ATTRIBUTE_INIT()
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
static const wchar_t * ZZ_CMAP()
static const int32_t ZZ_ACTION_PACKED_LENGTH
Definition StandardTokenizerImpl.h:42
static const int32_t ZZ_NO_MATCH
Definition StandardTokenizerImpl.h:70
static void ZZ_ROWMAP_INIT()
Translates a state to a row index in the transition table.
static const int32_t ZZ_ATTRIBUTE_PACKED_LENGTH
Definition StandardTokenizerImpl.h:79
static const int32_t ZZ_TRANS_LENGTH
Definition StandardTokenizerImpl.h:61
static const int32_t * ZZ_ACTION()
static IntArray _ZZ_ACTION
Translates DFA states to action switch labels.
Definition StandardTokenizerImpl.h:39
static void ZZ_CMAP_INIT()
Translates characters to character classes.
void reset(const ReaderPtr &r)
Resets the Tokenizer to a new Reader.
static const int32_t YYEOF
This character denotes the end of file.
Definition StandardTokenizerImpl.h:129
static const int32_t ZZ_ROWMAP_LENGTH
Definition StandardTokenizerImpl.h:51
ReaderPtr zzReader
The input device.
Definition StandardTokenizerImpl.h:86
static const int32_t ZZ_BUFFERSIZE
Initial size of the lookahead buffer.
Definition StandardTokenizerImpl.h:26
static IntArray _ZZ_ATTRIBUTE
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
Definition StandardTokenizerImpl.h:76
static const int32_t ZZ_CMAP_PACKED_LENGTH
Definition StandardTokenizerImpl.h:32
void zzScanError(int32_t errorCode)
Reports an error that occurred while scanning.
static const int32_t * ZZ_ATTRIBUTE()
String yytext()
Returns the text matched by the current regular expression.
static CharArray _ZZ_CMAP
Translates characters to character classes.
Definition StandardTokenizerImpl.h:29
void yypushback(int32_t number)
Pushes the specified amount of characters back into the input stream.
void yyreset(const ReaderPtr &reader)
Resets the scanner to read from a new input stream. Does not close the old reader.
static const int32_t ZZ_ATTRIBUTE_LENGTH
Definition StandardTokenizerImpl.h:78
static const int32_t ZZ_CMAP_LENGTH
Definition StandardTokenizerImpl.h:31
int32_t zzState
The current state of the DFA.
Definition StandardTokenizerImpl.h:89
bool zzAtBOL
zzAtBOL == true if the scanner is currently at the beginning of a line
Definition StandardTokenizerImpl.h:122
bool zzAtEOF
zzAtEOF == true if the scanner is at the EOF
Definition StandardTokenizerImpl.h:125
int32_t yylength()
Returns the length of the matched text region.
wchar_t yycharat(int32_t pos)
Returns the character at position pos from the matched text.
void getText(const TokenPtr &t)
Fills Lucene token with the current token text.
static const int32_t * ZZ_TRANS()
int32_t getNextToken()
Resumes scanning until the next regular expression is matched, the end of input is encountered or an ...
CharArray zzBuffer
This buffer contains the current text to be matched and is the source of the yytext() string.
Definition StandardTokenizerImpl.h:95
int32_t yystate()
Returns the current lexical state.
static const int32_t ZZ_PUSHBACK_2BIG
Definition StandardTokenizerImpl.h:71
int32_t zzPushbackPos
The text position at the last state to be included in yytext.
Definition StandardTokenizerImpl.h:101
static void ZZ_ACTION_INIT()
Translates DFA states to action switch labels.
bool zzRefill()
Refills the input buffer.
virtual ~StandardTokenizerImpl()
int32_t _yychar
The number of characters up to the start of the matched text.
Definition StandardTokenizerImpl.h:116
static IntArray _ZZ_TRANS
The transition table of the DFA.
Definition StandardTokenizerImpl.h:59
static const int32_t ZZ_UNKNOWN_ERROR
Definition StandardTokenizerImpl.h:69
int32_t zzStartRead
StartRead marks the beginning of the yytext() string in the buffer.
Definition StandardTokenizerImpl.h:107
static const wchar_t ZZ_TRANS_PACKED_0[]
Definition StandardTokenizerImpl.h:60
StandardTokenizerImpl(const ReaderPtr &in)
Creates a new scanner.
void getText(const TermAttributePtr &t)
Fills TermAttribute with the current token text.
static const wchar_t ZZ_ATTRIBUTE_PACKED_0[]
Definition StandardTokenizerImpl.h:77
static const wchar_t * ZZ_ERROR_MSG[]
Definition StandardTokenizerImpl.h:73
static const wchar_t ZZ_ROWMAP_PACKED_0[]
Definition StandardTokenizerImpl.h:50
static const wchar_t ZZ_CMAP_PACKED[]
Definition StandardTokenizerImpl.h:30
void yybegin(int32_t newState)
Enters a new lexical state.
static const int32_t YYINITIAL
Lexical states.
Definition StandardTokenizerImpl.h:132
static const int32_t * ZZ_ROWMAP()
static void ZZ_TRANS_INIT()
The transition table of the DFA.
static const wchar_t ZZ_ACTION_PACKED_0[]
Definition StandardTokenizerImpl.h:40
static const int32_t ZZ_TRANS_PACKED_LENGTH
Definition StandardTokenizerImpl.h:62
int32_t yyline
Number of newlines encountered up to the start of the matched text.
Definition StandardTokenizerImpl.h:113
static const int32_t ZZ_ROWMAP_PACKED_LENGTH
Definition StandardTokenizerImpl.h:52
Definition AbstractAllTermDocs.h:12
boost::shared_ptr< TermAttribute > TermAttributePtr
Definition LuceneTypes.h:58
boost::shared_ptr< Token > TokenPtr
Definition LuceneTypes.h:59
boost::shared_ptr< Reader > ReaderPtr
Definition LuceneTypes.h:547