hebmorph/org.gnit.lucenekmp.analysis.he/HebrewTokenizer

HebrewTokenizer

class HebrewTokenizer : Tokenizer

Tokenizes a given stream using HebMorph's Tokenizer, removes prefixes where possible, and tags Tokens with appropriate types where possible

Constructors

HebrewTokenizer

constructor(prefixes: MutableMap<String, Int>)

constructor(_prefixesTree: MutableMap<String, Int>, specialCases: DictRadix<Byte>?)

Types

object Companion

interface TOKEN_TYPES

Properties

attributeClassesIterator

val attributeClassesIterator: Iterator<Any>

attributeFactory

val attributeFactory: AttributeFactory

attributeImplsIterator

val attributeImplsIterator: Iterator<AttributeImpl>

Functions

fun <T : Attribute> addAttribute(attClass: KClass<T>): T

addAttributeImpl

fun addAttributeImpl(att: AttributeImpl)

fun captureState(): AttributeSource.State?

clearAttributes

fun clearAttributes()

cloneAttributes

fun cloneAttributes(): AttributeSource

open override fun close()

fun copyTo(target: AttributeSource)

open override fun end()

fun endAttributes()

open operator override fun equals(obj: Any?): Boolean

fun <T : Attribute> getAttribute(attClass: KClass<T>): T?

fun hasAttribute(attClass: KClass<out Attribute>): Boolean

fun hasAttributes(): Boolean

open override fun hashCode(): Int

open override fun incrementToken(): Boolean

fun isLegalPrefix(str: String): Boolean

reflectAsString

fun reflectAsString(prependAttClass: Boolean): String

fun reflectWith(reflector: AttributeReflector)

removeAllAttributes

fun removeAllAttributes()

open override fun reset()

fun restoreState(state: AttributeSource.State?)

fun setReader(input: Reader)

setSuffixForExactMatch

fun setSuffixForExactMatch(suffixForExactMatch: Char?)

open override fun toString(): String

tryStrippingPrefix

fun tryStrippingPrefix(word: String): String