ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

examples : switch retrieval to llama_encode (#13685)

Browse source 
* switch retrieval to llama_encode

* enable --no-warmup for retrieval
This commit is contained in:
Sigbjørn Skjæret 2025-05-21 16:57:38 +02:00 • committed by GitHub
parent eb0f5c28d3
commit 2aa777d86d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
common/arg.cpp
View file

@ -1678,7 +1678,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params) { [](common_params & params) {
params.warmup = false; params.warmup = false;
} }
).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING})); ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_RETRIEVAL}));
add_opt(common_arg( add_opt(common_arg(
{"--spm-infill"}, {"--spm-infill"},
string_format( string_format(

12
examples/retrieval/retrieval.cpp
View file

@ -81,14 +81,14 @@ static void batch_add_seq(llama_batch & batch, const std::vector<int32_t> & toke
} }
} }
static void batch_decode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) { static void batch_encode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) {
// clear previous kv_cache values (irrelevant for embeddings) // clear previous kv_cache values (irrelevant for embeddings)
llama_kv_self_clear(ctx); llama_kv_self_clear(ctx);
// run model // run model
LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq);
if (llama_decode(ctx, batch) < 0) { if (llama_encode(ctx, batch) < 0) {
LOG_ERR("%s : failed to decode\n", __func__); LOG_ERR("%s : failed to encode\n", __func__);
} }
for (int i = 0; i < batch.n_tokens; i++) { for (int i = 0; i < batch.n_tokens; i++) {
@ -233,7 +233,7 @@ int main(int argc, char ** argv) {
// encode if at capacity // encode if at capacity
if (batch.n_tokens + n_toks > n_batch) { if (batch.n_tokens + n_toks > n_batch) {
float * out = emb + p * n_embd; float * out = emb + p * n_embd;
batch_decode(ctx, batch, out, s, n_embd); batch_encode(ctx, batch, out, s, n_embd);
common_batch_clear(batch); common_batch_clear(batch);
p += s; p += s;
s = 0; s = 0;
@ -246,7 +246,7 @@ int main(int argc, char ** argv) {
// final batch // final batch
float * out = emb + p * n_embd; float * out = emb + p * n_embd;
batch_decode(ctx, batch, out, s, n_embd); batch_encode(ctx, batch, out, s, n_embd);
// save embeddings to chunks // save embeddings to chunks
for (int i = 0; i < n_chunks; i++) { for (int i = 0; i < n_chunks; i++) {
@ -267,7 +267,7 @@ int main(int argc, char ** argv) {
batch_add_seq(query_batch, query_tokens, 0); batch_add_seq(query_batch, query_tokens, 0);
std::vector<float> query_emb(n_embd, 0); std::vector<float> query_emb(n_embd, 0);
batch_decode(ctx, query_batch, query_emb.data(), 1, n_embd); batch_encode(ctx, query_batch, query_emb.data(), 1, n_embd);
common_batch_clear(query_batch); common_batch_clear(query_batch);