ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

llama : rename jina tokenizers to v2 (#7249)

Browse source 
* refactor: rename jina tokenizers to v2

* refactor: keep refactoring non-breaking
This commit is contained in:
Joan Fontanals 2024-05-13 10:35:14 +02:00 • committed by GitHub
parent b1f8af1886
commit 9aa672490c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 9 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -4424,7 +4424,9 @@ static void llm_load_vocab(
} else if (
tokenizer_pre == "gpt-2" ||
tokenizer_pre == "jina-es" ||
tokenizer_pre == "jina-de") {
tokenizer_pre == "jina-de" ||
tokenizer_pre == "jina-v2-es" ||
tokenizer_pre == "jina-v2-de") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
} else if (
tokenizer_pre == "refact") {