c642bc014c
* kv-cache : serparate recurrent vs non-recurrent impl (wip) ggml-ci * kv-cache : init -> contructor + add llama_memory_params ggml-ci * kv-cache : fix callback reference ggml-ci * context : llama_kv_cache -> llama_memory_i ggml-ci * context : move memory creation logic to model ggml-ci * llama : remove reference of memory during encode ggml-ci * kv-cache : hide padding details in the implementation ggml-ci * kv-cache : add ubatch_next() ggml-ci * context : simplify sbatch logic ggml-ci * kv-cache : hide defrag logic in the implementation ggml-ci * context : hide kv cache details in implementation ggml-ci * build : fix ggml-ci * cont : another fix ggml-ci * kv-cache : simplify interface (wip) ggml-ci * kv-cache : use separate KV cell structs for unified/recurrent ggml-ci * kv-cache : clean-up ggml-ci * model : better llama_model::create_model() signature ggml-ci * kv-cache : fix recurrent seq_rm() ggml-ci * kv-cache : replace `struct callbacks` with `llama_model &` ggml-ci * kv-cache : replace `struct graph_params` with `llama_context &` ggml-ci * kv-cache : fix offload check ggml-ci * context : avoid passing unique_ptr ggml-ci * kv-cache : avoid using the backends from the llama_context ref #13113 ggml-ci * kv-cache : more consistent debug logs [no ci] * kv-cache : do not pass the full llama_context for kv graphs ggml-ci * kv-cache : remove comment * kv-cache : ggml_rope_ext_inplace -> ggml_rope_ext ggml-ci * kv-cache : fix recurrent multi-user case ggml-ci * memory : remove comments [no ci]
89 lines
2.5 KiB
C++
89 lines
2.5 KiB
C++
#pragma once
|
|
|
|
#include "llama.h"
|
|
|
|
#include <array>
|
|
#include <vector>
|
|
|
|
// very similar to llama_batch,
|
|
// but has more metadata about sequences
|
|
struct llama_ubatch {
|
|
bool equal_seqs;
|
|
// TODO: whole_seqs for embeddings?
|
|
|
|
uint32_t n_tokens; // total tokens (n_seq_tokens * n_seqs)
|
|
uint32_t n_seq_tokens; // tokens per sequence
|
|
uint32_t n_seqs;
|
|
|
|
llama_token * token; // [n_tokens]
|
|
float * embd; // [n_embd, n_tokens]
|
|
llama_pos * pos; // [n_tokens]
|
|
int32_t * n_seq_id; // [n_seqs]
|
|
llama_seq_id ** seq_id; // [n_seqs]
|
|
int8_t * output; // [n_tokens]
|
|
};
|
|
|
|
struct llama_sbatch_seq {
|
|
int32_t n_seq_id;
|
|
|
|
llama_seq_id * seq_id;
|
|
|
|
size_t offset;
|
|
size_t length;
|
|
};
|
|
|
|
// sequence-length-aware batch splitting
|
|
struct llama_sbatch {
|
|
// tokens left in this batch
|
|
size_t n_tokens;
|
|
|
|
size_t n_embd;
|
|
|
|
bool logits_all; // TODO: remove once lctx.logits_all is removed too
|
|
|
|
// sorted indices into the batch
|
|
std::vector<int64_t> ids;
|
|
// batch indices of the output
|
|
std::vector<int64_t> out_ids;
|
|
std::vector<llama_sbatch_seq> seq;
|
|
|
|
const llama_batch * batch = nullptr;
|
|
|
|
// buffers for the ubatch
|
|
std::vector<llama_token> ubatch_token;
|
|
std::vector<float> ubatch_embd;
|
|
std::vector<llama_pos> ubatch_pos;
|
|
std::vector<int32_t> ubatch_n_seq_id;
|
|
std::vector<llama_seq_id *> ubatch_seq_id;
|
|
std::vector<int8_t> ubatch_output;
|
|
|
|
llama_ubatch reserve_ubatch(size_t n_ubatch, bool has_embd = false);
|
|
|
|
void add_seq_to_ubatch(llama_ubatch & ubatch, llama_sbatch_seq & seq, size_t length);
|
|
|
|
// simple split, unknown number of sequences of unequal lengths
|
|
llama_ubatch split_simple(size_t n_ubatch);
|
|
|
|
// make batches of equal-length sequences
|
|
llama_ubatch split_equal(size_t n_ubatch);
|
|
|
|
// sequence-wise split
|
|
llama_ubatch split_seq(size_t n_ubatch);
|
|
|
|
llama_sbatch() = default;
|
|
llama_sbatch(const llama_batch & batch, size_t n_embd, bool simple_split = false, bool logits_all = false);
|
|
};
|
|
|
|
// temporary allocate memory for the input batch if needed
|
|
struct llama_batch_allocr {
|
|
struct llama_batch batch;
|
|
|
|
std::array<llama_seq_id, 1> seq_id_0 = { 0 }; // default sequence id
|
|
std::vector<llama_pos> pos;
|
|
std::vector<int32_t> n_seq_id;
|
|
std::vector<llama_seq_id *> seq_id;
|
|
std::vector<int8_t> logits;
|
|
|
|
// optionally fulfill the batch returned by llama_batch_get_one
|
|
llama_batch_allocr(struct llama_batch in_batch, llama_pos p0);
|
|
};
|