From 4032ca406632212b075e3c61c2a4476128321410 Mon Sep 17 00:00:00 2001 From: Piotr Jasiukajtis Date: Sun, 25 May 2025 10:29:43 +0200 Subject: [PATCH] llama : add support for Qwen3 MoE tied word embeddings (#13768) --- src/llama-model.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 81b052e1..e99f5309 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2489,7 +2489,11 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i];