Lucene++ - a full-featured, c++ search engine
API Documentation
#include <StandardTokenizerImpl.h>
Public Member Functions | |
StandardTokenizerImpl (const ReaderPtr &in) | |
Creates a new scanner. | |
virtual | ~StandardTokenizerImpl () |
virtual String | getClassName () |
boost::shared_ptr< StandardTokenizerImpl > | shared_from_this () |
int32_t | yychar () |
void | reset (const ReaderPtr &r) |
Resets the Tokenizer to a new Reader. | |
void | getText (const TokenPtr &t) |
Fills Lucene token with the current token text. | |
void | getText (const TermAttributePtr &t) |
Fills TermAttribute with the current token text. | |
void | yyclose () |
Closes the input stream. | |
void | yyreset (const ReaderPtr &reader) |
Resets the scanner to read from a new input stream. Does not close the old reader. | |
int32_t | yystate () |
Returns the current lexical state. | |
void | yybegin (int32_t newState) |
Enters a new lexical state. | |
String | yytext () |
Returns the text matched by the current regular expression. | |
wchar_t | yycharat (int32_t pos) |
Returns the character at position pos from the matched text. | |
int32_t | yylength () |
Returns the length of the matched text region. | |
void | yypushback (int32_t number) |
Pushes the specified amount of characters back into the input stream. | |
int32_t | getNextToken () |
Resumes scanning until the next regular expression is matched, the end of input is encountered or an I/O- Error occurs. | |
![]() | |
virtual | ~LuceneObject () |
virtual void | initialize () |
Called directly after instantiation to create objects that depend on this object being fully constructed. | |
virtual LuceneObjectPtr | clone (const LuceneObjectPtr &other=LuceneObjectPtr()) |
Return clone of this object. | |
virtual int32_t | hashCode () |
Return hash code for this object. | |
virtual bool | equals (const LuceneObjectPtr &other) |
Return whether two objects are equal. | |
virtual int32_t | compareTo (const LuceneObjectPtr &other) |
Compare two objects. | |
virtual String | toString () |
Returns a string representation of the object. | |
![]() | |
virtual | ~LuceneSync () |
virtual SynchronizePtr | getSync () |
Return this object synchronize lock. | |
virtual LuceneSignalPtr | getSignal () |
Return this object signal. | |
virtual void | lock (int32_t timeout=0) |
Lock this object using an optional timeout. | |
virtual void | unlock () |
Unlock this object. | |
virtual bool | holdsLock () |
Returns true if this object is currently locked by current thread. | |
virtual void | wait (int32_t timeout=0) |
Wait for signal using an optional timeout. | |
virtual void | notifyAll () |
Notify all threads waiting for signal. | |
Static Public Member Functions | |
static String | _getClassName () |
Static Public Attributes | |
static const int32_t | YYEOF |
This character denotes the end of file. | |
static const int32_t | YYINITIAL |
Lexical states. | |
Protected Member Functions | |
bool | zzRefill () |
Refills the input buffer. | |
void | zzScanError (int32_t errorCode) |
Reports an error that occurred while scanning. | |
![]() | |
LuceneObject () | |
Static Protected Member Functions | |
static void | ZZ_CMAP_INIT () |
Translates characters to character classes. | |
static const wchar_t * | ZZ_CMAP () |
static void | ZZ_ACTION_INIT () |
Translates DFA states to action switch labels. | |
static const int32_t * | ZZ_ACTION () |
static void | ZZ_ROWMAP_INIT () |
Translates a state to a row index in the transition table. | |
static const int32_t * | ZZ_ROWMAP () |
static void | ZZ_TRANS_INIT () |
The transition table of the DFA. | |
static const int32_t * | ZZ_TRANS () |
static void | ZZ_ATTRIBUTE_INIT () |
ZZ_ATTRIBUTE[aState] contains the attributes of state aState. | |
static const int32_t * | ZZ_ATTRIBUTE () |
Protected Attributes | |
ReaderPtr | zzReader |
The input device. | |
int32_t | zzState |
The current state of the DFA. | |
int32_t | zzLexicalState |
The current lexical state. | |
CharArray | zzBuffer |
This buffer contains the current text to be matched and is the source of the yytext() string. | |
int32_t | zzMarkedPos |
The text position at the last accepting state. | |
int32_t | zzPushbackPos |
The text position at the last state to be included in yytext. | |
int32_t | zzCurrentPos |
The current text position in the buffer. | |
int32_t | zzStartRead |
StartRead marks the beginning of the yytext() string in the buffer. | |
int32_t | zzEndRead |
EndRead marks the last character in the buffer, that has been read from input. | |
int32_t | yyline |
Number of newlines encountered up to the start of the matched text. | |
int32_t | _yychar |
The number of characters up to the start of the matched text. | |
int32_t | yycolumn |
The number of characters from the last newline up to the start of the matched text. | |
bool | zzAtBOL |
zzAtBOL == true if the scanner is currently at the beginning of a line | |
bool | zzAtEOF |
zzAtEOF == true if the scanner is at the EOF | |
![]() | |
SynchronizePtr | objectLock |
LuceneSignalPtr | objectSignal |
Static Protected Attributes | |
static const int32_t | ZZ_BUFFERSIZE |
Initial size of the lookahead buffer. | |
static CharArray | _ZZ_CMAP |
Translates characters to character classes. | |
static const wchar_t | ZZ_CMAP_PACKED [] |
static const int32_t | ZZ_CMAP_LENGTH |
static const int32_t | ZZ_CMAP_PACKED_LENGTH |
static IntArray | _ZZ_ACTION |
Translates DFA states to action switch labels. | |
static const wchar_t | ZZ_ACTION_PACKED_0 [] |
static const int32_t | ZZ_ACTION_LENGTH |
static const int32_t | ZZ_ACTION_PACKED_LENGTH |
static IntArray | _ZZ_ROWMAP |
Translates a state to a row index in the transition table. | |
static const wchar_t | ZZ_ROWMAP_PACKED_0 [] |
static const int32_t | ZZ_ROWMAP_LENGTH |
static const int32_t | ZZ_ROWMAP_PACKED_LENGTH |
static IntArray | _ZZ_TRANS |
The transition table of the DFA. | |
static const wchar_t | ZZ_TRANS_PACKED_0 [] |
static const int32_t | ZZ_TRANS_LENGTH |
static const int32_t | ZZ_TRANS_PACKED_LENGTH |
static const int32_t | ZZ_UNKNOWN_ERROR |
static const int32_t | ZZ_NO_MATCH |
static const int32_t | ZZ_PUSHBACK_2BIG |
static const wchar_t * | ZZ_ERROR_MSG [] |
static IntArray | _ZZ_ATTRIBUTE |
ZZ_ATTRIBUTE[aState] contains the attributes of state aState. | |
static const wchar_t | ZZ_ATTRIBUTE_PACKED_0 [] |
static const int32_t | ZZ_ATTRIBUTE_LENGTH |
static const int32_t | ZZ_ATTRIBUTE_PACKED_LENGTH |
Lucene::StandardTokenizerImpl::StandardTokenizerImpl | ( | const ReaderPtr & | in | ) |
Creates a new scanner.
in | the Reader to read input from. |
|
virtual |
|
inlinestatic |
|
inlinevirtual |
int32_t Lucene::StandardTokenizerImpl::getNextToken | ( | ) |
Resumes scanning until the next regular expression is matched, the end of input is encountered or an I/O- Error occurs.
void Lucene::StandardTokenizerImpl::getText | ( | const TermAttributePtr & | t | ) |
Fills TermAttribute with the current token text.
void Lucene::StandardTokenizerImpl::getText | ( | const TokenPtr & | t | ) |
Fills Lucene token with the current token text.
void Lucene::StandardTokenizerImpl::reset | ( | const ReaderPtr & | r | ) |
|
inline |
void Lucene::StandardTokenizerImpl::yybegin | ( | int32_t | newState | ) |
Enters a new lexical state.
newState | the new lexical state. |
int32_t Lucene::StandardTokenizerImpl::yychar | ( | ) |
wchar_t Lucene::StandardTokenizerImpl::yycharat | ( | int32_t | pos | ) |
Returns the character at position pos from the matched text.
It is equivalent to yytext()[pos], but faster
pos | the position of the character to fetch. A value from 0 to yylength() - 1. |
void Lucene::StandardTokenizerImpl::yyclose | ( | ) |
Closes the input stream.
int32_t Lucene::StandardTokenizerImpl::yylength | ( | ) |
Returns the length of the matched text region.
void Lucene::StandardTokenizerImpl::yypushback | ( | int32_t | number | ) |
Pushes the specified amount of characters back into the input stream.
They will be read again by then next call of the scanning method
number | the number of characters to be read again. This number must not be greater than yylength() |
void Lucene::StandardTokenizerImpl::yyreset | ( | const ReaderPtr & | reader | ) |
Resets the scanner to read from a new input stream. Does not close the old reader.
All internal variables are reset, the old input stream cannot be reused (internal buffer is discarded and lost). Lexical state is set to ZZ_INITIAL.
reader | the new input stream. |
int32_t Lucene::StandardTokenizerImpl::yystate | ( | ) |
Returns the current lexical state.
String Lucene::StandardTokenizerImpl::yytext | ( | ) |
Returns the text matched by the current regular expression.
|
staticprotected |
|
staticprotected |
Translates DFA states to action switch labels.
|
staticprotected |
|
staticprotected |
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
|
staticprotected |
|
staticprotected |
Translates characters to character classes.
|
staticprotected |
|
staticprotected |
Translates a state to a row index in the transition table.
|
staticprotected |
|
staticprotected |
The transition table of the DFA.
|
protected |
Refills the input buffer.
|
protected |
Reports an error that occurred while scanning.
In a well-formed scanner (no or only correct usage of yypushback(int32_t) and a match-all fallback rule) this method will only be called with things that "Can't Possibly Happen". If this method is called, something is seriously wrong.
Usual syntax/scanner level error handling should be done in error fallback rules.
errorCode | The code of the errormessage to display. |
|
protected |
The number of characters up to the start of the matched text.
|
staticprotected |
Translates DFA states to action switch labels.
|
staticprotected |
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
|
staticprotected |
Translates characters to character classes.
|
staticprotected |
Translates a state to a row index in the transition table.
|
staticprotected |
The transition table of the DFA.
|
protected |
The number of characters from the last newline up to the start of the matched text.
|
static |
This character denotes the end of file.
|
static |
Lexical states.
|
protected |
Number of newlines encountered up to the start of the matched text.
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
Initial size of the lookahead buffer.
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
staticprotected |
|
protected |
zzAtBOL == true if the scanner is currently at the beginning of a line
|
protected |
zzAtEOF == true if the scanner is at the EOF
|
protected |
This buffer contains the current text to be matched and is the source of the yytext() string.
|
protected |
The current text position in the buffer.
|
protected |
EndRead marks the last character in the buffer, that has been read from input.
|
protected |
The current lexical state.
|
protected |
The text position at the last accepting state.
|
protected |
The text position at the last state to be included in yytext.
|
protected |
The input device.
|
protected |
StartRead marks the beginning of the yytext() string in the buffer.
|
protected |
The current state of the DFA.