common/org.gnit.lucenekmp.analysis.commongrams/CommonGramsFilter

CommonGramsFilter

class CommonGramsFilter(input: TokenStream, commonWords: CharArraySet?) : TokenFilter

Construct bigrams for frequently occurring terms while indexing. Single terms are still indexed too, with bigrams overlaid. This is achieved through the use of PositionIncrementAttribute.setPositionIncrement. Bigrams have a type of GRAM_TYPE Example:

input:"the quick brown fox"
output:|"the","the-quick"|"brown"|"fox"|
"the-quick" has a position increment of 0 so it is in the same position as "the" "the-quick" has a term.type() of "gram"

Constructors

CommonGramsFilter

constructor(input: TokenStream, commonWords: CharArraySet?)

Types

Companion

object Companion

Properties

attributeClassesIterator

val attributeClassesIterator: Iterator<Any>

attributeFactory

val attributeFactory: AttributeFactory

attributeImplsIterator

val attributeImplsIterator: Iterator<AttributeImpl>

Functions

addAttribute

fun <T : Attribute> addAttribute(attClass: KClass<T>): T

addAttributeImpl

fun addAttributeImpl(att: AttributeImpl)

captureState

fun captureState(): AttributeSource.State?

clearAttributes

fun clearAttributes()

cloneAttributes

fun cloneAttributes(): AttributeSource

open override fun close()

copyTo

fun copyTo(target: AttributeSource)

end

open override fun end()

endAttributes

fun endAttributes()

equals

open operator override fun equals(obj: Any?): Boolean

getAttribute

fun <T : Attribute> getAttribute(attClass: KClass<T>): T?

hasAttribute

fun hasAttribute(attClass: KClass<out Attribute>): Boolean

hasAttributes

fun hasAttributes(): Boolean

hashCode

open override fun hashCode(): Int

incrementToken

open override fun incrementToken(): Boolean

Inserts bigrams for common words into a token stream. For each input token, output the token. If the token and/or the following token are in the list of common words also output a bigram with position increment 0 and type="gram"

reflectAsString

fun reflectAsString(prependAttClass: Boolean): String

reflectWith

fun reflectWith(reflector: AttributeReflector)

removeAllAttributes

fun removeAllAttributes()

reset

open override fun reset()

restoreState

fun restoreState(state: AttributeSource.State?)

toString

open override fun toString(): String

unwrap

open override fun unwrap(): TokenStream