Batch Decoding Operations. More...

#include "common.hpp"
#include <algorithm>
#include <array>
#include <cstdint>
#include <llama/llama.h>
#include <memory>
#include <span>
#include <stdexcept>
#include <string>
#include <vector>

Classes
struct	lloyal::decode::DecodeError

struct	lloyal::decode::EachItem
	Input item for decode::each — one token for one sequence. More...

struct	lloyal::decode::ScatterItem
	Input item for decode::scatter — multiple tokens for one sequence. More...

struct	lloyal::decode::Scratch
	Reusable scratch buffers for multi-sequence batch construction. More...

struct	lloyal::decode::EmbdItem
	Input item for decode::embd — embedding rows for one sequence. More...

struct	lloyal::decode::Segment
	One run of input in a heterogeneous prefill. More...

struct	lloyal::decode::SegmentSource
	Supplies the segment sequence for one branch's prefill. More...

class	lloyal::decode::TokenSource
	A token span as a stream of one TEXT segment, so text is placed by the rail like any other input. More...

struct	lloyal::decode::SegmentProgress
	How far one input of a stream has landed: a reading of progress. More...

struct	lloyal::decode::LandedSegment
	A segment of a stream once it is in the KV: what moved, for the caller's books. More...

struct	lloyal::decode::StreamReading
	How a context reads what is placed into it: the one read of its traits that placement decides from. More...

struct	lloyal::decode::PackedChunk
	A chunk of item indices produced by bin_pack() More...

Namespaces
namespace	lloyal
	Boundary Tracker Stub for OSS liblloyal.

namespace	lloyal::decode

namespace	lloyal::decode::shape_guard

namespace	lloyal::decode::detail

Functions
void	lloyal::decode::shape_guard::llama_batch_has_seven_members ()

void	lloyal::decode::detail::batch_clear (llama_batch &batch)
	The two helpers llama.cpp's common library used to provide for a llama_batch_init()-allocated batch: reset the count, append one token targeting one sequence.

void	lloyal::decode::detail::batch_add (llama_batch &batch, llama_token id, llama_pos pos, llama_seq_id seq_id, bool logits)

bool	lloyal::decode::leaves_branch_intact (int32_t rc, bool partial)
	llama_decode failure carrying the raw return code

void	lloyal::decode::require_fits_batch (llama_context ctx, int32_t n_tokens, const char who)
	The context's batch is a hard limit: the pinned llama.cpp asserts `n_tokens <= n_batch` inside llama_decode — an abort, not an rc.

int	lloyal::decode::many (llama_context ctx, const llama_token tokens, int32_t n_tokens, int32_t n_past, int32_t n_batch, llama_seq_id seq_id=0, int32_t *n_committed=nullptr)
	Decode multiple tokens into the KV cache with auto-chunking.

int	lloyal::decode::many (llama_context *ctx, const std::vector< llama_token > &tokens, int32_t n_past, int32_t n_batch, llama_seq_id seq_id=0)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

int	lloyal::decode::one (llama_context *ctx, llama_token tok, llama_pos pos, llama_seq_id seq_id=0, bool want_logits=true)
	Decode a single token into the KV cache.

void	lloyal::decode::require_row_width (llama_context ctx, int32_t n_embd_inp, const char who)
	Rows must be exactly the model's input width: a wrong-but-positive width starts later rows mid-row and reads past the caller's allocation, with nothing to signal it.

int	lloyal::decode::each (llama_context ctx, const EachItem items, int32_t n, Scratch &scratch)
	Decode one token per sequence in a single llama_decode() call.

int	lloyal::decode::each (llama_context *ctx, const std::vector< EachItem > &items, Scratch &scratch)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

int	lloyal::decode::scatter (llama_context ctx, const ScatterItem items, int32_t n, Scratch &scratch)
	Decode multiple tokens per sequence in a single llama_decode() call.

int	lloyal::decode::scatter (llama_context *ctx, const std::vector< ScatterItem > &items, Scratch &scratch)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

int	lloyal::decode::embd (llama_context ctx, const EmbdItem &item, int32_t n_batch, Scratch &scratch, int32_t n_committed=nullptr)
	Decode pre-computed embedding rows into one sequence's KV cache.

StreamReading	lloyal::decode::reading_of (llama_context *ctx)

void	lloyal::decode::detail::require_segment (const Segment &seg, size_t i)
	A segment the source yielded, checked before anything is sized from it or handed back to the source: the source is a public extension point, so its segments are untrusted input.

template<typename OnLanded >
void	lloyal::decode::detail::segments_each (llama_context *ctx, llama_seq_id seq, llama_pos base, SegmentSource &source, int32_t rows_n_batch, Scratch &scratch, std::vector< llama_pos > &positions, OnLanded &&landed)
	Each segment its own dispatch: the shape of a prefill into a sequence that keeps what it was given.

template<typename OnLanded >
void	lloyal::decode::detail::segments_whole (llama_context *ctx, const StreamReading &reading, llama_seq_id seq, llama_pos base, SegmentSource &source, std::vector< llama_pos > &positions, OnLanded &&landed)
	The whole stream in one dispatch: one llama_batch_ext, tokens and rows as entries in order, one llama_process.

template<typename OnLanded >
void	lloyal::decode::segments (llama_context *ctx, llama_seq_id seq, llama_pos base, SegmentSource &source, int32_t rows_n_batch, Scratch &scratch, std::vector< llama_pos > &positions, OnLanded &&landed)
	Place a source's segments into one sequence, in order.

std::vector< PackedChunk >	lloyal::decode::bin_pack (const std::span< const llama_token > *items, int32_t n, int32_t n_batch)
	Greedy first-fit bin-packing of token spans into n_batch-sized chunks.

Detailed Description

Batch Decoding Operations.

Wraps llama.cpp decode APIs with batch management, chunking logic, and orchestration primitives. Provides both batched and single-token decode operations.

API naming follows this grid:

                Single Sequence       Multi Sequence
               ┌─────────────────┬─────────────────┐

Single Token │ decode::one │ decode::each │ ├─────────────────┼─────────────────┤ Multi Token │ decode::many │ decode::scatter│ ├─────────────────┼─────────────────┤ Embedding Rows │ decode::embd │ — │ └─────────────────┴─────────────────┘

A stream of text and rows goes through decode::segments, which composes these a segment per dispatch, or places the whole stream as one mixed batch on a context that reads it only whole (see StreamReading).

Single-sequence primitives auto-chunk internally (many, embd); the multi-sequence ones do not — packing across sequences is a policy decision that lives in BranchStore (see bin_pack).

Batches allocated by llama_batch_init() are filled by detail::batch_clear/batch_add.

Logit Indexing: How llama_get_logits_ith() Maps to Batch Positions

llama.cpp packs logits into a dense output buffer — only tokens with batch.logits[i] = true get logits computed. The internal output_ids vector translates batch positions to packed rows:

Batch:      [tok0, tok1, tok2, tok3, tok4, tok5, tok6, tok7]
logits[]:   [  0,    0,    0,    0,    1,    0,    0,    1 ]
 
output_ids: [ -1,   -1,   -1,   -1,    0,   -1,   -1,    1]
                                       ^                 ^
                                    row 0             row 1
 
llama_get_logits_ith(ctx, 4)  → output_ids[4] = 0  → logits + 0*n_vocab  ✓
llama_get_logits_ith(ctx, 7)  → output_ids[7] = 1  → logits + 1*n_vocab  ✓
llama_get_logits_ith(ctx, 0)  → output_ids[0] = -1 → throws (no logits)
llama_get_logits_ith(ctx, -1) → n_outputs - 1 = 1  → logits + 1*n_vocab  (last output)

Callers always pass batch positions, not packed indices. The output_ids indirection handles the translation. Negative indices bypass output_ids entirely: -1 means the last output row, -2 the second-to-last, etc.

This matters for logit capture in BranchStore:

Decode pattern	logits flag	Access index
decode::one	Last token only	`-1` (sole output)
decode::many	Last token of final chunk only	`-1` (sole output of last dispatch)
decode::each	All items (1:1 with branches)	`i` (batch pos = item index)
decode::scatter	Last token per item	`cursor + n_tokens[k] - 1`

For decode::many, each chunk is a separate llama_decode() call that resets the output buffer. Only the final chunk's last token has logits, so after the last dispatch n_outputs = 1 and -1 yields row 0.

Definition in file decode.hpp.

Classes

Namespaces

Functions

Detailed Description

Logit Indexing: How llama_get_logits_ith() Maps to Batch Positions