ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

clip : remove boi/eoi embeddings for GLM-edge model (#13081)

Browse source
This commit is contained in:
Xuan-Son Nguyen 2025-04-24 22:17:04 +02:00 • committed by GitHub
parent 226251ed56
commit 13be08daf9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 6 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

5
examples/llava/mtmd.cpp
View file

@ -186,6 +186,11 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
marker_modified = "<start_of_image>" + ctx->image_marker + "<end_of_image>";
string_replace_all(prompt_modified, ctx->image_marker, marker_modified);
} else if (proj_type == PROJECTOR_TYPE_GLM_EDGE) {
// <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
marker_modified = "<|begin_of_image|>" + ctx->image_marker + "<|end_of_image|>";
string_replace_all(prompt_modified, ctx->image_marker, marker_modified);
} else if (proj_type == PROJECTOR_TYPE_IDEFICS3) {
// https://github.com/huggingface/transformers/blob/a42ba80fa520c784c8f11a973ca9034e5f859b79/src/transformers/models/idefics3/processing_idefics3.py#L192-L215
marker_modified = "<fake_token_around_image><global-img>" + ctx->image_marker + "<fake_token_around_image>";