ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

llama : correct rms norm for llama 4 (#12882)

Browse source
This commit is contained in:
Xuan-Son Nguyen 2025-04-11 08:49:50 +02:00 • committed by GitHub
parent 0fed24c347
commit 8b91d5355a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/llama-model.cpp
View file

@ -4440,8 +4440,8 @@ struct llm_build_llama : public llm_graph_context {
if (arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) { if (arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) {
// Llama4TextL2Norm // Llama4TextL2Norm
Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6); Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6); Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
cb(Qcur, "Qcur_normed", il); cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il); cb(Kcur, "Kcur_normed", il);
} }