core/org.gnit.lucenekmp.analysis.standard/StandardTokenizer/Companion

Companion

object Companion

Properties

const val ALPHANUM: Int = 0

Alpha/numeric token type

const val EMOJI: Int = 7

Emoji token type.

const val HANGUL: Int = 6

Hangul token type

const val HIRAGANA: Int = 4

Hiragana token type

const val IDEOGRAPHIC: Int = 3

Ideographic token type

const val KATAKANA: Int = 5

Katakana token type

MAX_TOKEN_LENGTH_LIMIT

const val MAX_TOKEN_LENGTH_LIMIT: Int

Absolute maximum sized token

const val NUM: Int = 1

Numeric token type

SOUTHEAST_ASIAN

const val SOUTHEAST_ASIAN: Int = 2

Southeast Asian token type

val TOKEN_TYPES: Array<String>

String token types that correspond to token type int constants