Interface RerankOptions

Options for Rerank context creation

interface RerankOptions {
    modelPath: string;
    nCtx?: number;
    nSeqMax?: number;
    typeK?: KvCacheType;
    typeV?: KvCacheType;
}

Index

Properties

modelPath nCtx? nSeqMax? typeK? typeV?

Properties

modelPath

modelPath: string

Path to reranker .gguf model

`Optional`nCtx

nCtx?: number

Context window size (default: 4096)

`Optional`nSeqMax

nSeqMax?: number

Max prompts per GPU dispatch (default: 8)

`Optional`typeK

typeK?: KvCacheType

KV cache key quantization (default: 'q4_0')

`Optional`typeV

typeV?: KvCacheType

KV cache value quantization (default: 'q4_0')