ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

llama : improve sep token handling (#14272)

Browse source
This commit is contained in:
Sigbjørn Skjæret 2025-06-20 14:04:09 +02:00 • committed by GitHub
parent e28c1b93fd
commit 88fc854b4b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 161 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

1
include/llama.h
View file

@ -1044,6 +1044,7 @@ extern "C" {
LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);
LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);