hebmorph/org.gnit.lucenekmp.analysis.he/HebMorphTokenizer

HebMorphTokenizer

class HebMorphTokenizer(input: Reader, hebrewPrefixes: MutableMap<String, Int>, specialCases: DictRadix<Byte>? = null)

Constructors

HebMorphTokenizer

constructor(input: Reader, hebrewPrefixes: MutableMap<String, Int>, specialCases: DictRadix<Byte>? = null)

Types

object Companion

object TokenType

Functions

fun addSpecialCase(token: String)

clearSpecialCases

fun clearSpecialCases()

getLengthInSource

fun getLengthInSource(): Int

fun getOffset(): Int

getSuffixForExactMatch

fun getSuffixForExactMatch(): Char?

fun nextToken(tokenString: Reference<String>): Int

fun reset(_input: Reader)

setSuffixForExactMatch

fun setSuffixForExactMatch(suffixForExactMatch: Char?)