7#ifndef KEYWORDTOKENIZER_H
8#define KEYWORDTOKENIZER_H
35 void init(int32_t bufferSize);
#define LUCENE_CLASS(Name)
Definition LuceneObject.h:24
Emits the entire input as a single token.
Definition KeywordTokenizer.h:15
OffsetAttributePtr offsetAtt
Definition KeywordTokenizer.h:32
void init(int32_t bufferSize)
virtual bool incrementToken()
Consumers (ie., IndexWriter) use this method to advance the stream to the next token....
virtual ~KeywordTokenizer()
bool done
Definition KeywordTokenizer.h:29
KeywordTokenizer(const ReaderPtr &input)
KeywordTokenizer(const AttributeFactoryPtr &factory, const ReaderPtr &input, int32_t bufferSize)
virtual void reset()
Resets this stream to the beginning. This is an optional operation, so subclasses may or may not impl...
TermAttributePtr termAtt
Definition KeywordTokenizer.h:31
static const int32_t DEFAULT_BUFFER_SIZE
Definition KeywordTokenizer.h:27
KeywordTokenizer(const ReaderPtr &input, int32_t bufferSize)
int32_t finalOffset
Definition KeywordTokenizer.h:30
KeywordTokenizer(const AttributeSourcePtr &source, const ReaderPtr &input, int32_t bufferSize)
virtual void end()
This method is called by the consumer after the last token has been consumed, after incrementToken() ...
A Tokenizer is a TokenStream whose input is a Reader.
Definition Tokenizer.h:20
Definition AbstractAllTermDocs.h:12
boost::shared_ptr< AttributeSource > AttributeSourcePtr
Definition LuceneTypes.h:520
boost::shared_ptr< TermAttribute > TermAttributePtr
Definition LuceneTypes.h:58
boost::shared_ptr< OffsetAttribute > OffsetAttributePtr
Definition LuceneTypes.h:40
boost::shared_ptr< Reader > ReaderPtr
Definition LuceneTypes.h:547
boost::shared_ptr< AttributeFactory > AttributeFactoryPtr
Definition LuceneTypes.h:519