Classes
class	Branch

struct	BranchState
	Consolidated mutable state for a single branch. More...

class	BranchStore
	Handle table and batched decode orchestrator for branch management. More...

struct	CachedSamplingParams
	The resolved sampling params a branch's chain was built from. More...

struct	DecodeEachItem
	Item for decode_each: one token per branch. More...

struct	DecodeScatterItem
	Item for decode_scatter: variable tokens per branch. More...

struct	DecodeSegmentsResult
	What a heterogeneous prefill consumed. More...

struct	ForkOpts
	Options controlling what state fork() clones from source to child. More...

struct	GrammarEntry
	RAII entry for a grammar sampler in the registry. More...

struct	KvPressure
	Snapshot of KV cache pressure from BranchStore. More...

struct	SamplerChainEntry
	RAII entry for a sampler chain in the registry. More...

Typedefs
using	BranchHandle = uint32_t
	Opaque handle to a branch slot.

using	SamplerChainHandle = int32_t
	Handle to a sampler chain in BranchStore's registry (0 = invalid/none)

using	GrammarHandle = int32_t
	Handle to a grammar sampler in BranchStore's registry (0 = invalid/none)

using	MetricsHandle = int32_t
	Handle to a metrics tracker in BranchStore's registry (0 = invalid/none)

Functions
uint16_t	handle_index (BranchHandle h)
	Extract slot index from a branch handle.

uint16_t	handle_generation (BranchHandle h)
	Extract generation counter from a branch handle.

BranchHandle	make_handle (uint16_t index, uint16_t generation)
	Construct a branch handle from index and generation.

template<SamplingParamsLike P>
CachedSamplingParams	snapshot_params (const P &p, uint32_t absent_seed)
	Resolve sampling params into the record a chain is built from.

uint32_t	clock_seed ()
	A fresh seed for a chain created without one.

void	require_distinct_handles (std::span< const BranchHandle > handles, const char *who)
	A handle may appear at most once per batched call: each item's start position is read from its branch when the batch is built and advances only after dispatch, so two items for one branch would collide on position — two tokens on one cell, or overlapping runs.

template<SamplingParamsLike P>
BranchHandle	create (llama_context ctx, const llama_model model, BranchStore &s, llama_pos start_pos, const P &params, int n_batch=DEFAULT_N_BATCH, const char grammar_str=nullptr, boundaries::BoundaryTracker boundary_tracker=nullptr)
	Create a new branch with sampler chain, optional grammar, and metrics.

BranchHandle	fork (BranchHandle source, BranchStore &s, ForkOpts opts={})
	Fork a branch into a new independent sequence.

void	set_logit_bias (BranchHandle handle, const llama_logit_bias *biases, size_t n_biases, BranchStore &s)

void	clear_logit_bias (BranchHandle handle, BranchStore &s)
	Clear all logit biases from a branch.

void	set_steer (BranchHandle handle, std::function< void(llama_token_data_array &)> steer_fn, BranchStore &s)

void	clear_steer (BranchHandle handle, BranchStore &s)
	Clear the steer callback from a branch.

template<SamplingParamsLike P>
void	set_sampler_params (BranchHandle handle, const P &params, BranchStore &s)
	Replace a branch's sampler chain with new parameters.

void	set_grammar (BranchHandle handle, const llama_model model, const char grammar_str, BranchStore &s)
	Replace a branch's grammar constraint.

void	set_grammar_lazy (BranchHandle handle, const llama_model model, const char grammar_str, const std::vector< std::string > &trigger_patterns, const std::vector< llama_token > &trigger_tokens, BranchStore &s)
	Set lazy grammar on a branch (unconstrained until trigger fires)

void	set_grammar_order (BranchHandle handle, bool sample_first, BranchStore &s)
	Choose how sample() consults the attached grammar.

void	prune (BranchHandle handle, BranchStore &s)
	Prune a leaf branch (RESTRICT — throws if children exist)

void	pruneSubtree (BranchHandle h, BranchStore &s)
	Prune a branch and all descendants (CASCADE — iterative post-order)

void	reseed_sampler (BranchHandle handle, uint32_t seed, BranchStore &s)
	Reseed a branch's sampling PRNG, the way forks are made to diverge.

void	fast_forward (BranchHandle parent, BranchHandle child, BranchStore &s)
	Fast-forward `parent` to `child`; see BranchStore::fast_forward()

bool	is_poisoned (BranchHandle handle, BranchStore &s)
	Whether a failed decode left this branch's KV ahead of its books.

void	force_snapshot_logits (BranchHandle handle, BranchStore &s)
	Force-copy the shared llama.cpp logits buffer into this branch's private snapshot.

void	prefill (BranchHandle handle, const llama_token *tokens, size_t n_tokens, BranchStore &s)
	Decode multiple tokens and capture logits atomically (prompt prefill)

void	step (BranchHandle handle, llama_token token, BranchStore &s)
	Decode a single token and capture logits (generation step)

const float *	get_logits (BranchHandle handle, BranchStore &s)
	Get the branch's captured logits snapshot.

void	set_logits (BranchHandle handle, std::span< const float > data, BranchStore &s)
	Overwrite a branch's logits_snapshot with caller-provided values.

void	get_logits_at (BranchHandle handle, std::span< const int32_t > indices, std::span< float > out, BranchStore &s)
	Read selected logit values from a branch's snapshot.

llama_token	sample (BranchHandle handle, BranchStore &s)
	Sample a token from the branch's captured logits.

void	accept_token (BranchHandle handle, llama_token token, BranchStore &s)
	Accept a sampled token, advancing grammar and sampler state.

void	apply_grammar (BranchHandle handle, float *logits, int n_vocab, BranchStore &s)
	Apply grammar constraints to an external logits buffer.

std::vector< std::pair< llama_token, float > >	get_legal_priors (BranchHandle handle, BranchStore &s)
	Get grammar-legal tokens with renormalized probabilities.

float	get_legal_logsumexp (BranchHandle handle, BranchStore &s)
	Compute log-sum-exp over grammar-legal logits.

bool	is_token_legal (BranchHandle handle, llama_token token, BranchStore &s)
	Check if a token is legal under grammar constraints.

float	get_token_prior_assume_legal (BranchHandle handle, llama_token token, float logsumexp, BranchStore &s)
	Compute prior probability for a token known to be grammar-legal.

float	get_token_prior (BranchHandle handle, llama_token token, float logsumexp, BranchStore &s)
	Compute prior probability for a token, checking grammar legality first.

llama_pos	get_position (BranchHandle handle, BranchStore &s)
	Get the branch's current decode position.

llama_pos	get_fork_head (BranchHandle handle, BranchStore &s)
	Get the branch's fork head (parent position at fork time)

float	get_perplexity (BranchHandle handle, BranchStore &s)
	Get model-level perplexity (from raw logits)

float	get_sampling_perplexity (BranchHandle handle, BranchStore &s)
	Get sampling-level perplexity (from filtered distribution)

float	get_last_sampling_prior (BranchHandle handle, BranchStore &s)
	Get the last sampled token's prior from the filtered distribution.

int	get_n_vocab (BranchHandle handle, BranchStore &s)
	Get the branch's vocabulary size.

Variables
constexpr BranchHandle	INVALID_HANDLE = 0
	Null handle sentinel.

constexpr llama_seq_id	NO_LEASE = kv::NO_LEASE
	Branch has no KV residency.

constexpr int	DEFAULT_N_BATCH = 512
	Default batch size for decode operations.

constexpr uint32_t	GEN_SHIFT = 16
	Bit shift for generation field.

constexpr uint32_t	INDEX_MASK = 0xFFFF
	Mask for slot index field.

Typedef Documentation

◆ BranchHandle

using lloyal::branch::BranchHandle = typedef uint32_t

Opaque handle to a branch slot.

Encoded as (generation << 16) | index:

Upper 16 bits: generation counter (prevents ABA bugs on slot reuse)
Lower 16 bits: slot index (max 65535 branches)
Value 0 is reserved as the invalid/null handle

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 100 of file branch.hpp.

◆ GrammarHandle

using lloyal::branch::GrammarHandle = typedef int32_t

Handle to a grammar sampler in BranchStore's registry (0 = invalid/none)

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 162 of file branch.hpp.

◆ MetricsHandle

using lloyal::branch::MetricsHandle = typedef int32_t

Handle to a metrics tracker in BranchStore's registry (0 = invalid/none)

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 165 of file branch.hpp.

◆ SamplerChainHandle

using lloyal::branch::SamplerChainHandle = typedef int32_t

Handle to a sampler chain in BranchStore's registry (0 = invalid/none)

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 159 of file branch.hpp.

Function Documentation

◆ accept_token()

void lloyal::branch::accept_token	(	BranchHandle	handle,
		llama_token	token,
		BranchStore &	s
	)

inline

Accept a sampled token, advancing grammar and sampler state.

Updates:

Grammar parser state (if grammar is attached)
Sampler chain penalty tracking (repetition/frequency penalties)
Model-level perplexity (from raw logits, if available)
Sampling-level perplexity (from filtered candidate distribution)

Parameters

handle	Branch that produced the token
token	Token ID returned by sample()
s	Branch store

Note: Safe to call with invalid handle (silent no-op).

Definition at line 2632 of file branch.hpp.

◆ apply_grammar()

void lloyal::branch::apply_grammar	(	BranchHandle	handle,
		float *	logits,
		int	n_vocab,
		BranchStore &	s
	)

inline

Apply grammar constraints to an external logits buffer.

Sets logits of grammar-illegal tokens to -INFINITY in the provided buffer. Uses the branch's internal candidates_buffer as scratch space when vocab sizes match; allocates a temporary buffer otherwise.

Parameters

handle	Branch with grammar to apply
logits	Logits buffer to modify in place (n_vocab floats)
n_vocab	Number of entries in the logits buffer
s	Branch store

Note: No-op if handle is invalid or branch has no grammar attached.

Definition at line 2701 of file branch.hpp.

◆ clear_logit_bias()

void lloyal::branch::clear_logit_bias	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Clear all logit biases from a branch.

Parameters

handle	Branch to modify
s	Branch store

Exceptions

std::runtime_error if handle is invalid

Definition at line 1981 of file branch.hpp.

◆ clear_steer()

void lloyal::branch::clear_steer	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Clear the steer callback from a branch.

Parameters

handle	Branch to modify
s	Branch store

Exceptions

std::runtime_error if handle is invalid

Definition at line 2046 of file branch.hpp.

◆ clock_seed()

uint32_t lloyal::branch::clock_seed ( )

inline

A fresh seed for a chain created without one.

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 267 of file branch.hpp.

◆ create()

template<SamplingParamsLike P>

BranchHandle lloyal::branch::create	(	llama_context *	ctx,
		const llama_model *	model,
		BranchStore &	s,
		llama_pos	start_pos,
		const P &	params,
		int	n_batch = `DEFAULT_N_BATCH`,
		const char *	grammar_str = `nullptr`,
		boundaries::BoundaryTracker *	boundary_tracker = `nullptr`
	)

inline

Create a new branch with sampler chain, optional grammar, and metrics.

Allocates a slot + KV lease from the store, initializes the sampler chain from params, optionally attaches a GBNF grammar and boundary tracker, and pre-allocates logits/candidates buffers sized to the model's vocabulary.

Template Parameters

P	Any type satisfying the SamplingParamsLike concept

Parameters

ctx	Llama context (not owned, must outlive branch)
model	Llama model (not owned, used for vocab size and sampler init)
s	Branch store to allocate from
start_pos	Starting decode position (typically prompt length after prefill)
params	Sampling parameters (temperature, top_k, top_p, penalties, etc.)
n_batch	Batch size for decode operations (default 512)
grammar_str	GBNF grammar string, or nullptr for unconstrained generation
boundary_tracker	Boundary detector (ownership transferred), or nullptr

Returns: Valid BranchHandle, or INVALID_HANDLE on failure

See also: prune() to free with KV cleanup, pruneSubtree() for CASCADE

Definition at line 1740 of file branch.hpp.

◆ fast_forward()

void lloyal::branch::fast_forward	(	BranchHandle	parent,
		BranchHandle	child,
		BranchStore &	s
	)

inline

Fast-forward parent to child; see BranchStore::fast_forward()

Definition at line 2276 of file branch.hpp.

◆ force_snapshot_logits()

void lloyal::branch::force_snapshot_logits	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Force-copy the shared llama.cpp logits buffer into this branch's private snapshot.

Copies the logits from the llama_context into the branch's internal buffer without performing a decode.

Warning: The shared buffer contains logits from the LAST llama_decode() call. If another branch (or batched decode) ran since this branch's last decode, the snapshot will contain wrong logits. Only call this immediately after a single-branch decode for this handle. Prefer prefill()/step() which capture atomically.

Parameters

handle	Branch to capture logits for
s	Branch store

Exceptions

std::runtime_error if handle is invalid, vocab size is zero, or no logits are available (no prior decode with logits enabled)

Note: Sets has_logits = true, enabling sample() and get_logits().

Definition at line 2308 of file branch.hpp.

◆ fork()

BranchHandle lloyal::branch::fork	(	BranchHandle	source,
		BranchStore &	s,
		ForkOpts	opts = `{}`
	)

inline

Fork a branch into a new independent sequence.

Allocates a slot + KV lease, deep copies source state under the new seq_id. Records parent→child topology edge.

Cloned state:

KV cache (via kv::seq_cp)
Sampler chain (penalties, PRNG, filters)
Grammar (parser state)
Boundary tracker
Metrics (model + sampling perplexity)
Logits snapshot and logit bias (logits skipped if opts.clone_logits=false)

NOT cloned:

steer_fn (may capture references — call set_steer() on the child if needed)

Parameters

source	Handle of the branch to fork from
s	Branch store
opts	Fine-grained control over what to clone (see ForkOpts)

Returns: Handle to the new child branch, or INVALID_HANDLE on failure

Definition at line 1845 of file branch.hpp.

◆ get_fork_head()

llama_pos lloyal::branch::get_fork_head	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get the branch's fork head (parent position at fork time)

Parameters

handle	Branch handle
s	Branch store

Returns: Fork head position (0 for root branches or invalid handles)

Definition at line 3011 of file branch.hpp.

◆ get_last_sampling_prior()

float lloyal::branch::get_last_sampling_prior	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get the last sampled token's prior from the filtered distribution.

Returns P(token) from the post-filter sampling distribution. This is the correct prior for UCT-family algorithms since it matches what was actually sampled.

Parameters

handle	Branch handle
s	Branch store

Returns: Probability of last sampled token in [0, 1], or 0 if unavailable

Definition at line 3066 of file branch.hpp.

◆ get_legal_logsumexp()

float lloyal::branch::get_legal_logsumexp	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Compute log-sum-exp over grammar-legal logits.

Returns log(sum(exp(logit_i))) over tokens that pass grammar constraints. Use for efficient per-token prior computation: P(token) = exp(logit[token] - logsumexp)

Numerically stable (max-subtraction trick).

Parameters

handle	Branch with captured logits and optional grammar
s	Branch store

Returns: Log-sum-exp value, or -INFINITY if no legal tokens or invalid state

See also: get_token_prior_assume_legal() for O(1) per-token prior using this value

Definition at line 2836 of file branch.hpp.

◆ get_legal_priors()

std::vector< std::pair< llama_token, float > > lloyal::branch::get_legal_priors	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get grammar-legal tokens with renormalized probabilities.

Returns (token, probability) pairs for tokens that pass grammar constraints. Probabilities are softmax-normalized over the legal set only (sum to 1.0).

Essential for policy priors in tree search: priors must only cover legal moves.

Parameters

handle	Branch with captured logits and optional grammar
s	Branch store

Returns: Vector of (token_id, probability) pairs, empty if no logits or no legal tokens

Note: If no grammar is attached, all tokens with finite logits are included.

Definition at line 2757 of file branch.hpp.

◆ get_logits()

const float * lloyal::branch::get_logits	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get the branch's captured logits snapshot.

Returns a pointer to the internal logits buffer (n_vocab floats). Only valid after force_snapshot_logits(), prefill(), or step().

Parameters

handle	Branch to read logits from
s	Branch store

Returns: Pointer to n_vocab floats, or nullptr if no logits captured

Definition at line 2406 of file branch.hpp.

◆ get_logits_at()

void lloyal::branch::get_logits_at	(	BranchHandle	handle,
		std::span< const int32_t >	indices,
		std::span< float >	out,
		BranchStore &	s
	)

inline

Read selected logit values from a branch's snapshot.

Reads logit values at the given vocab indices from state->logits_snapshot into the provided output buffer. Use when only a few logit values are needed (rerank yes/no probe, entailment scoring, semantic-entropy reads) instead of copying n_vocab floats out via get_logits().

Reads from state->logits_snapshot (per-branch persistent snapshot) — NOT from logits::get(ctx, ...) (which uses flattened cursor indexing and whose pointer's lifetime ends at the next decode). Handle-addressed, decode-order -immune, concurrency-safe by virtue of being branch-local.

Parameters

handle	Branch to read from
indices	Vocab indices to read (Int32Array from JS through N-API)
out	Output span; same length as indices
s	Branch store

Exceptions

std::runtime_error if handle invalid, no logits captured, sizes mismatch, or any index is out of [0, n_vocab)

Definition at line 2476 of file branch.hpp.

◆ get_n_vocab()

int lloyal::branch::get_n_vocab	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get the branch's vocabulary size.

Parameters

handle	Branch handle
s	Branch store

Returns: Vocabulary size, or 0 if handle is invalid

Definition at line 3103 of file branch.hpp.

◆ get_perplexity()

float lloyal::branch::get_perplexity	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get model-level perplexity (from raw logits)

Returns perplexity computed from the full logit distribution before any sampler filtering. For the distribution actually sampled from, use get_sampling_perplexity().

Parameters

handle	Branch handle
s	Branch store

Returns: Model perplexity, or INFINITY if no tokens accepted

Definition at line 3027 of file branch.hpp.

◆ get_position()

llama_pos lloyal::branch::get_position	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get the branch's current decode position.

Parameters

handle	Branch handle
s	Branch store

Returns: Token position, or -1 if handle is invalid

Definition at line 2999 of file branch.hpp.

◆ get_sampling_perplexity()

float lloyal::branch::get_sampling_perplexity	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Get sampling-level perplexity (from filtered distribution)

Returns perplexity from the distribution actually sampled from (after top-k/p/temp/penalties). Useful for policy priors and monitoring sampler chain impact.

Parameters

handle	Branch handle
s	Branch store

Returns: Sampling-level perplexity, or INFINITY if no tokens accepted

Definition at line 3047 of file branch.hpp.

◆ get_token_prior()

float lloyal::branch::get_token_prior	(	BranchHandle	handle,
		llama_token	token,
		float	logsumexp,
		BranchStore &	s
	)

inline

Compute prior probability for a token, checking grammar legality first.

O(grammar_complexity) — uses is_token_legal() before computing the prior. Safe for ad-hoc callers who don't know whether the token is grammar-legal.

Parameters

handle	Branch with captured logits and optional grammar
token	Token ID to compute prior for
logsumexp	Pre-computed value from get_legal_logsumexp()
s	Branch store

Returns: Probability in [0, 1], or 0 if token is illegal

Note: For search inner loops, prefer get_token_prior_assume_legal() since sample() already enforces grammar constraints.

Definition at line 2979 of file branch.hpp.

◆ get_token_prior_assume_legal()

float lloyal::branch::get_token_prior_assume_legal	(	BranchHandle	handle,
		llama_token	token,
		float	logsumexp,
		BranchStore &	s
	)

inline

Compute prior probability for a token known to be grammar-legal.

O(1) operation — use in search inner loops where sample() already enforced grammar. Does NOT validate grammar legality; caller must ensure token is legal.

Parameters

handle	Branch with captured logits
token	Token ID (must be legal under grammar)
logsumexp	Pre-computed value from get_legal_logsumexp()
s	Branch store

Returns: Probability in [0, 1], or 0 if state is invalid

See also: get_token_prior() for a safe version that checks grammar legality

Definition at line 2948 of file branch.hpp.

◆ handle_generation()

uint16_t lloyal::branch::handle_generation ( BranchHandle h )

inline

Extract generation counter from a branch handle.

Parameters

h	Branch handle

Returns: Generation counter (upper 16 bits)

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 142 of file branch.hpp.

◆ handle_index()

uint16_t lloyal::branch::handle_index ( BranchHandle h )

inline

Extract slot index from a branch handle.

Parameters

h	Branch handle

Returns: Slot index (lower 16 bits)

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 133 of file branch.hpp.

◆ is_poisoned()

bool lloyal::branch::is_poisoned	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Whether a failed decode left this branch's KV ahead of its books.

Returns: true if poisoned (prune it and replay); false if intact or invalid

Definition at line 2284 of file branch.hpp.

◆ is_token_legal()

bool lloyal::branch::is_token_legal	(	BranchHandle	handle,
		llama_token	token,
		BranchStore &	s
	)

inline

Check if a token is legal under grammar constraints.

Uses a 1-element candidate array for O(grammar_complexity) check instead of the O(n_vocab) full scan used by get_legal_priors().

Parameters

handle	Branch with optional grammar
token	Token ID to check
s	Branch store

Returns: true if token is legal (or no grammar attached), false if illegal

Definition at line 2897 of file branch.hpp.

◆ make_handle()

BranchHandle lloyal::branch::make_handle	(	uint16_t	index,
		uint16_t	generation
	)

inline

Construct a branch handle from index and generation.

Parameters

index	Slot index (0–65535)
generation	Generation counter

Returns: Encoded branch handle

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 152 of file branch.hpp.

◆ prefill()

void lloyal::branch::prefill	(	BranchHandle	handle,
		const llama_token *	tokens,
		size_t	n_tokens,
		BranchStore &	s
	)

inline

Decode multiple tokens and capture logits atomically (prompt prefill)

Feeds tokens through the model in n_batch-sized chunks, advances the branch position, and snapshots logits. After this call, sample() and get_logits() are available.

Parameters

handle	Branch to decode into
tokens	Array of token IDs
n_tokens	Number of tokens in the array
s	Branch store

Exceptions

std::runtime_error if handle is invalid, decode fails, or logits capture fails

Note: For single-token decode, prefer step() (zero heap allocation).

Definition at line 2336 of file branch.hpp.

◆ prune()

void lloyal::branch::prune	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Prune a leaf branch (RESTRICT — throws if children exist)

Evicts the KV lease and frees all resources via BranchStore::release(). If the branch has children, throws — use pruneSubtree() for CASCADE.

Parameters

handle	Branch to prune (INVALID_HANDLE is a safe no-op)
s	Branch store

Exceptions

std::runtime_error if branch has children

Definition at line 2221 of file branch.hpp.

◆ pruneSubtree()

void lloyal::branch::pruneSubtree	(	BranchHandle	h,
		BranchStore &	s
	)

inline

Prune a branch and all descendants (CASCADE — iterative post-order)

Traverses the subtree rooted at h, collecting all descendants, then prunes leaves-first so RESTRICT on prune() always passes.

Parameters

h	Root of subtree to prune
s	Branch store

Definition at line 2238 of file branch.hpp.

◆ require_distinct_handles()

void lloyal::branch::require_distinct_handles	(	std::span< const BranchHandle >	handles,
		const char *	who
	)

inline

A handle may appear at most once per batched call: each item's start position is read from its branch when the batch is built and advances only after dispatch, so two items for one branch would collide on position — two tokens on one cell, or overlapping runs.

The rule is stated here once for every path that batches by handle (decode_each, decode_scatter, and a binding's cohort that dispatches one branch at a time). One pass; a cohort is at most n_seq_max wide. INVALID_HANDLE entries are skipped — an empty item occupies no cells and cannot collide with anything.

Exceptions

std::runtime_error naming both indices of the first repeat

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 436 of file branch.hpp.

◆ reseed_sampler()

void lloyal::branch::reseed_sampler	(	BranchHandle	handle,
		uint32_t	seed,
		BranchStore &	s
	)

inline

Reseed a branch's sampling PRNG, the way forks are made to diverge.

Only a stochastic chain (one ending in dist) has a PRNG; reseeding a greedy chain would change nothing, so it is left alone. The cached params record the new seed, so set_sampler_params() compares against the chain as it is.

Parameters

handle	Branch to reseed
seed	New seed
s	Branch store

Exceptions

std::runtime_error if handle is invalid

Definition at line 2262 of file branch.hpp.

◆ sample()

llama_token lloyal::branch::sample	(	BranchHandle	handle,
		BranchStore &	s
	)

inline

Sample a token from the branch's captured logits.

Applies modifiers in order: Grammar → Logit Bias → Steer → Sampler Chain, then selects a token. With grammar_sample_first the grammar moves to the end as a check of the one chosen token, and the full order runs only when that check refuses it. Also records filtered candidates for metrics.

Requires prior force_snapshot_logits(), prefill(), or step().

Parameters

handle	Branch to sample from
s	Branch store

Returns: Sampled token ID, or -1 if no logits captured or sampling fails

Note: Call accept_token() after sampling to advance grammar and penalty state.

Definition at line 2519 of file branch.hpp.

◆ set_grammar()

void lloyal::branch::set_grammar	(	BranchHandle	handle,
		const llama_model *	model,
		const char *	grammar_str,
		BranchStore &	s
	)

inline

Replace a branch's grammar constraint.

Frees the old grammar (if any) and attaches a new one. Pass nullptr or empty string to remove grammar constraints entirely.

Parameters

handle	Branch to modify
model	Llama model (for vocab)
grammar_str	GBNF grammar string, or nullptr to remove
s	Branch store

Exceptions

std::runtime_error if handle is invalid

Definition at line 2117 of file branch.hpp.

◆ set_grammar_lazy()

void lloyal::branch::set_grammar_lazy	(	BranchHandle	handle,
		const llama_model *	model,
		const char *	grammar_str,
		const std::vector< std::string > &	trigger_patterns,
		const std::vector< llama_token > &	trigger_tokens,
		BranchStore &	s
	)

inline

Set lazy grammar on a branch (unconstrained until trigger fires)

Replaces any existing grammar. The lazy grammar accepts all tokens until a trigger pattern or token fires, then constrains subsequent generation.

Parameters

handle	Branch handle
model	Llama model (for vocab)
grammar_str	GBNF grammar string
trigger_patterns	Regex patterns that activate the grammar
trigger_tokens	Token IDs that activate the grammar
s	Branch store

Exceptions

std::runtime_error if handle is invalid

Definition at line 2158 of file branch.hpp.

◆ set_grammar_order()

void lloyal::branch::set_grammar_order	(	BranchHandle	handle,
		bool	sample_first,
		BranchStore &	s
	)

inline

Choose how sample() consults the attached grammar.

Mask-first (the default) tests every vocabulary entry against the grammar before the sampler chain runs, which is exact under top-k/top-p/min-p but costs a walk over the whole vocabulary per token. Sample-first runs the chain on the raw candidates, asks the grammar about the one chosen token, and only when it is refused falls back to the mask-first order for that token. Inside a free-text span of a grammar the pick is almost always legal, so the mask is almost never built. The order is reset to mask-first whenever a grammar is attached, so it has to be chosen after set_grammar()/set_grammar_lazy().

Parameters

handle	Branch to configure
sample_first	true for sample-then-check, false for mask-first
s	Branch store

Definition at line 2200 of file branch.hpp.

◆ set_logit_bias()

void lloyal::branch::set_logit_bias	(	BranchHandle	handle,
		const llama_logit_bias *	biases,
		size_t	n_biases,
		BranchStore &	s
	)

inline

Definition at line 1954 of file branch.hpp.

◆ set_logits()

void lloyal::branch::set_logits	(	BranchHandle	handle,
		std::span< const float >	data,
		BranchStore &	s
	)

inline

Overwrite a branch's logits_snapshot with caller-provided values.

Copies data.size() floats into state->logits_snapshot. The branch's KV state is unchanged — only the cached next-token distribution is replaced. After this call, sample() / produce() on this branch reads the supplied values (then runs the branch's sampler chain on top: grammar, logit_bias, steer, temperature, etc.).

Companion to get_logits(): read out, compute, write back. Use cases include distributions computed in JS/TS, save/restore patterns across decision points, and custom merge rules that aren't covered by BranchStore::merge_logits.

Parameters

handle	Branch to write to
data	Span of n_vocab floats (must match branch's vocab size)
s	Branch store

Exceptions

std::runtime_error if handle invalid or data length mismatches n_vocab

Definition at line 2436 of file branch.hpp.

◆ set_sampler_params()

template<SamplingParamsLike P>

void lloyal::branch::set_sampler_params	(	BranchHandle	handle,
		const P &	params,
		BranchStore &	s
	)

inline

Replace a branch's sampler chain with new parameters.

Memoized: if the new params match the cached snapshot, this is a no-op. Otherwise frees the old chain and creates a new one. An absent seed keeps the seed the chain has — a call that names only a temperature compares equal when nothing else changed, and a rebuild draws on the same seed, so a seeded branch stays the branch it was seeded as.

Primary use case: Entropy-based Dynamic Temperature (EDT), where temperature changes per-token based on model uncertainty.

Template Parameters

P	Any type satisfying the SamplingParamsLike concept

Parameters

handle	Branch to modify
params	New sampling parameters; an absent seed keeps the chain's
s	Branch store

Exceptions

std::runtime_error if handle is invalid

Definition at line 2080 of file branch.hpp.

◆ set_steer()

void lloyal::branch::set_steer	(	BranchHandle	handle,
		std::function< void(llama_token_data_array &)>	steer_fn,
		BranchStore &	s
	)

inline

Definition at line 2023 of file branch.hpp.

◆ snapshot_params()

template<SamplingParamsLike P>

CachedSamplingParams lloyal::branch::snapshot_params	(	const P &	p,
		uint32_t	absent_seed
	)

inline

Resolve sampling params into the record a chain is built from.

Every field takes the same default as sampler::create_chain(), except the seed, which has no default of its own: the caller says what an absent seed means. At creation it is a fresh clock seed; at set_sampler_params() it is the seed the chain already has, so a call that names no seed keeps it.

Parameters

p	Any SamplingParamsLike type; fields may be optional
absent_seed	The seed an absent `p.seed` resolves to

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 250 of file branch.hpp.

◆ step()

void lloyal::branch::step	(	BranchHandle	handle,
		llama_token	token,
		BranchStore &	s
	)

inline

Decode a single token and capture logits (generation step)

Uses decode::one() which maintains a thread_local batch — heap-allocated once per thread, reused across calls. No per-call allocation.

Parameters

handle	Branch to decode into
token	Token ID to decode
s	Branch store

Exceptions

std::runtime_error if handle is invalid, decode fails, or logits capture fails

Definition at line 2375 of file branch.hpp.

Variable Documentation

◆ DEFAULT_N_BATCH

constexpr int lloyal::branch::DEFAULT_N_BATCH = 512

constexpr

Default batch size for decode operations.

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 104 of file branch.hpp.

◆ GEN_SHIFT

constexpr uint32_t lloyal::branch::GEN_SHIFT = 16

constexpr

Bit shift for generation field.

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 105 of file branch.hpp.

◆ INDEX_MASK

constexpr uint32_t lloyal::branch::INDEX_MASK = 0xFFFF

constexpr

Mask for slot index field.

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 106 of file branch.hpp.

◆ INVALID_HANDLE

constexpr BranchHandle lloyal::branch::INVALID_HANDLE = 0

constexpr

Null handle sentinel.

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 102 of file branch.hpp.

◆ NO_LEASE

constexpr llama_seq_id lloyal::branch::NO_LEASE = kv::NO_LEASE

constexpr

Branch has no KV residency.

Examples: /home/runner/work/liblloyal/liblloyal/include/lloyal/branch.hpp.

Definition at line 103 of file branch.hpp.

Classes

Typedefs

Functions

Variables

Typedef Documentation

◆ BranchHandle

◆ GrammarHandle

◆ MetricsHandle

◆ SamplerChainHandle

Function Documentation

◆ accept_token()

◆ apply_grammar()

◆ clear_logit_bias()

◆ clear_steer()

◆ clock_seed()

◆ create()

◆ fast_forward()

◆ force_snapshot_logits()

◆ fork()

◆ get_fork_head()

◆ get_last_sampling_prior()

◆ get_legal_logsumexp()

◆ get_legal_priors()

◆ get_logits()

◆ get_logits_at()

◆ get_n_vocab()

◆ get_perplexity()

◆ get_position()

◆ get_sampling_perplexity()

◆ get_token_prior()

◆ get_token_prior_assume_legal()

◆ handle_generation()

◆ handle_index()

◆ is_poisoned()

◆ is_token_legal()

◆ make_handle()

◆ prefill()

◆ prune()

◆ pruneSubtree()

◆ require_distinct_handles()

◆ reseed_sampler()

◆ sample()

◆ set_grammar()

◆ set_grammar_lazy()

◆ set_grammar_order()

◆ set_logit_bias()

◆ set_logits()

◆ set_sampler_params()

◆ set_steer()

◆ snapshot_params()

◆ step()

Variable Documentation

◆ DEFAULT_N_BATCH

◆ GEN_SHIFT

◆ INDEX_MASK

◆ INVALID_HANDLE

◆ NO_LEASE