common/org.gnit.lucenekmp.analysis.classic/ClassicTokenizerImpl

ClassicTokenizerImpl

class ClassicTokenizerImpl(in: Reader)

This class implements the classic lucene StandardTokenizer up until 3.0

Constructors

constructor(in: Reader)

Types

Companion

object Companion

Properties

val ACRONYM: Int

val ACRONYM_DEP: Int

val ALPHANUM: Int

val APOSTROPHE: Int

val CJ: Int

COMPANY

val COMPANY: Int

val EMAIL: Int

HOST

val HOST: Int

NUM

val NUM: Int

TOKEN_TYPES

val TOKEN_TYPES: Array<String>

Functions

getNextToken

fun getNextToken(): Int

Resumes scanning until the next regular expression is matched, the end of input is encountered or an I/O-Error occurs.

getText

fun getText(t: CharTermAttribute)

Fills CharTermAttribute with the current token text.

setBufferSize

fun setBufferSize(numChars: Int)

yyatEOF

fun yyatEOF(): Boolean

Returns whether the scanner has reached the end of the reader it reads from.

yybegin

fun yybegin(newState: Int)

Enters a new lexical state.

yychar

fun yychar(): Int

yycharat

fun yycharat(position: Int): Char

Returns the character at the given position from the matched text.

yyclose

fun yyclose()

Closes the input reader.

yylength

fun yylength(): Int

How many characters were matched.

yypushback

fun yypushback(number: Int)

Pushes the specified amount of characters back into the input stream.

yyreset

fun yyreset(reader: Reader)

Resets the scanner to read from a new input stream.

yystate

fun yystate(): Int

Returns the current lexical state.

yytext

fun yytext(): String

Returns the text matched by the current regular expression.