ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

clip : revert the change of BOI/EOI token for GLM-edge (⚠️ breaking change) (#13259)

Browse source
This commit is contained in:
Xuan-Son Nguyen 2025-05-03 20:07:54 +02:00 • committed by GitHub
parent 3bf785f3ef
commit 36667c8edc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 22 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tools/llava/mtmd.cpp
View file

@ -189,11 +189,6 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
marker_modified = "<start_of_image>" + ctx->image_marker + "<end_of_image>";
string_replace_all(prompt_modified, ctx->image_marker, marker_modified);
} else if (proj_type == PROJECTOR_TYPE_GLM_EDGE) {
// <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
marker_modified = "<|begin_of_image|>" + ctx->image_marker + "<|end_of_image|>";
string_replace_all(prompt_modified, ctx->image_marker, marker_modified);
} else if (proj_type == PROJECTOR_TYPE_IDEFICS3) {
// https://github.com/huggingface/transformers/blob/a42ba80fa520c784c8f11a973ca9034e5f859b79/src/transformers/models/idefics3/processing_idefics3.py#L192-L215
marker_modified = "<fake_token_around_image><global-img>" + ctx->image_marker + "<fake_token_around_image>";
@ -213,6 +208,7 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
}
// llava-1.5, llava-1.6, Yi-VL, Yi-34B, granite: don't need to add prefix and suffix
// for glm-edge, BOI and EOI token's embeddings are not present in the text model
std::vector<std::string> parts = string_split_str(prompt_modified, ctx->image_marker);
output.clear();