ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

llama : auto-batch preparation (#13845)

Browse source 
* llama : auto-batch

ggml-ci

* context : simplify if branching
This commit is contained in:
Georgi Gerganov 2025-05-31 12:55:57 +03:00 • committed by GitHub
parent 51fa76f172
commit 3f55f781f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 67 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tools/server/server.cpp
View file

@ -3431,7 +3431,7 @@ struct server_context {
// retry with half the batch size to try to find a free slot in the KV cache
n_batch /= 2;
SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size - try increasing it via the context size or enable defragmentation, i = %d, n_batch = %d, ret = %d\n", i, n_batch, ret);
SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size, i = %d, n_batch = %d, ret = %d\n", i, n_batch, ret);
continue; // continue loop of n_batch
}