From 558a764713468f26f5a163d25a22100c9a04a48f Mon Sep 17 00:00:00 2001
From: City <125218114+city96@users.noreply.github.com>
Date: Fri, 25 Apr 2025 14:38:34 +0200
Subject: [PATCH] Force FP32 compute in GLM4 FFN Down (#13101)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Force FP32 compute in cuBLAS GEMM

* Revert "Force FP32 compute in cuBLAS GEMM"

This reverts commit 6efd872732159ab88ee7b3c1d77ba5ebc83079bd.

* Force F32 compute in GLM4 ffn down

* Edit comment to clarify issue

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

---------

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
---
 src/llama-graph.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index a85e9728..b52e3f62 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -803,6 +803,10 @@ ggml_tensor * llm_graph_context::build_ffn(
 
     if (down) {
         cur = build_lora_mm(down, cur);
+        if (arch == LLM_ARCH_GLM4) {
+            // GLM4 seems to have numerical issues with half-precision accumulators
+            ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
+        }
     }
 
     if (down_b) {