ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

server : add server parameters for draft model cache type (#13782)

Browse source 
Co-authored-by: aa956 <27946957+aa956@users.noreply.github.com>
This commit is contained in:
aa956 2025-06-19 16:01:03 +03:00 • committed by GitHub
parent 456af35eb7
commit d67341dc18
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 33 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tools/server/server.cpp
View file

@ -1969,10 +1969,8 @@ struct server_context {
params_dft.n_ctx = params_base.speculative.n_ctx == 0 ? params_base.n_ctx / params_base.n_parallel : params_base.speculative.n_ctx;
params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers;
params_dft.n_parallel = 1;
// force F16 KV cache for the draft model for extra performance
params_dft.cache_type_k = GGML_TYPE_F16;
params_dft.cache_type_v = GGML_TYPE_F16;
params_dft.cache_type_k = params_base.speculative.cache_type_k;
params_dft.cache_type_v = params_base.speculative.cache_type_v;
llama_init_dft = common_init_from_params(params_dft);