nori/org.gnit.lucenekmp.analysis.ko/KoreanTokenizer

KoreanTokenizer

Tokenizer for Korean that uses morphological analysis.

Constructors

constructor()

Creates a new KoreanTokenizer with default parameters.

constructor(factory: AttributeFactory, userDictionary: UserDictionary?, mode: KoreanTokenizer.DecompoundMode, outputUnknownUnigrams: Boolean)

Create a new KoreanTokenizer using the system and unknown dictionaries shipped with Lucene.

constructor(factory: AttributeFactory, userDictionary: UserDictionary?, mode: KoreanTokenizer.DecompoundMode, outputUnknownUnigrams: Boolean, discardPunctuation: Boolean)

Create a new KoreanTokenizer using the system and unknown dictionaries shipped with Lucene.

constructor(factory: AttributeFactory, systemDictionary: TokenInfoDictionary, unkDictionary: UnknownDictionary, connectionCosts: ConnectionCosts, userDictionary: UserDictionary?, mode: KoreanTokenizer.DecompoundMode, outputUnknownUnigrams: Boolean, discardPunctuation: Boolean)

Create a new KoreanTokenizer supplying a custom system dictionary and unknown dictionary.

Types

Companion

object Companion

DecompoundMode

enum DecompoundMode : Enum<KoreanTokenizer.DecompoundMode>

Decompound mode: this determines how the tokenizer handles POS.Type.COMPOUND, POS.Type.INFLECT and POS.Type.PREANALYSIS tokens.

Properties

attributeClassesIterator

val attributeClassesIterator: Iterator<Any>

attributeFactory

val attributeFactory: AttributeFactory

attributeImplsIterator

val attributeImplsIterator: Iterator<AttributeImpl>

Functions

addAttribute

fun <T : Attribute> addAttribute(attClass: KClass<T>): T

addAttributeImpl

fun addAttributeImpl(att: AttributeImpl)

captureState

fun captureState(): AttributeSource.State?

clearAttributes

fun clearAttributes()

cloneAttributes

fun cloneAttributes(): AttributeSource

open override fun close()

copyTo

fun copyTo(target: AttributeSource)

end

open override fun end()

endAttributes

fun endAttributes()

equals

open operator override fun equals(obj: Any?): Boolean

getAttribute

fun <T : Attribute> getAttribute(attClass: KClass<T>): T?

hasAttribute

fun hasAttribute(attClass: KClass<out Attribute>): Boolean

hasAttributes

fun hasAttributes(): Boolean

hashCode

open override fun hashCode(): Int

incrementToken

open override fun incrementToken(): Boolean

reflectAsString

fun reflectAsString(prependAttClass: Boolean): String

reflectWith

fun reflectWith(reflector: AttributeReflector)

removeAllAttributes

fun removeAllAttributes()

reset

open override fun reset()

restoreState

fun restoreState(state: AttributeSource.State?)

setGraphvizFormatter

fun setGraphvizFormatter(dotOut: GraphvizFormatter<KoMorphData>)

Expert: set this to produce graphviz (dot) output of the Viterbi lattice

setReader

fun setReader(input: Reader)

toString

open override fun toString(): String