extra/org.gnit.lucenekmp.analysis.km/GraphemeClusterTokenizer

GraphemeClusterTokenizer

class GraphemeClusterTokenizer : Tokenizer

Tokenizes a string in Khmer grapheme clusters (not phonetic syllables), for instance: "ខ្ញុំចង់ធ្វើការ" will be tokenized as "ខ្ញុំ", "ច", "ង់", "ធ្វើ", "កា", "រ", not "ខ្ញុំ", "ចង់", "ធ្វើ", "ការ". It uses a simple state machine to do so.

Constructors

GraphemeClusterTokenizer

constructor()

Properties

attributeClassesIterator

val attributeClassesIterator: Iterator<Any>

attributeFactory

val attributeFactory: AttributeFactory

attributeImplsIterator

val attributeImplsIterator: Iterator<AttributeImpl>

val CHCAT_BASE: Int = 1

val CHCAT_COENG: Int = 3

val CHCAT_DIGIT: Int = 4

val CHCAT_IGNORE: Int = 5

val CHCAT_INSIDE: Int = 2

DEFAULT_MAX_WORD_LEN

val DEFAULT_MAX_WORD_LEN: Int = 255

val ST_AFTERCOENG: Int = 2

val ST_AFTERDIGIT: Int = 3

val ST_INIT: Int = 4

val ST_INSIDESYL: Int = 1

Functions

fun <T : Attribute> addAttribute(attClass: KClass<T>): T

addAttributeImpl

fun addAttributeImpl(att: AttributeImpl)

fun captureState(): AttributeSource.State?

fun category(c: Int): Int

clearAttributes

fun clearAttributes()

cloneAttributes

fun cloneAttributes(): AttributeSource

open override fun close()

fun copyTo(target: AttributeSource)

open override fun end()

fun endAttributes()

open operator override fun equals(obj: Any?): Boolean

fun <T : Attribute> getAttribute(attClass: KClass<T>): T?

fun hasAttribute(attClass: KClass<out Attribute>): Boolean

fun hasAttributes(): Boolean

open override fun hashCode(): Int

open override fun incrementToken(): Boolean

reflectAsString

fun reflectAsString(prependAttClass: Boolean): String

fun reflectWith(reflector: AttributeReflector)

removeAllAttributes

fun removeAllAttributes()

open override fun reset()

fun restoreState(state: AttributeSource.State?)

fun setReader(input: Reader)

open override fun toString(): String