ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

gguf-py : fix Qwen3-Embedding eos token (#14314)

Browse source
This commit is contained in:
Sigbjørn Skjæret 2025-06-21 18:12:05 +02:00 • committed by GitHub
parent bb16041cae
commit aa0ef5c578
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 10 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

10
gguf-py/gguf/vocab.py
View file

@ -197,6 +197,16 @@ class SpecialVocab:
if special_last := tmpl_single[-1].get('SpecialToken', {}).get('id'): if special_last := tmpl_single[-1].get('SpecialToken', {}).get('id'):
if not tokenizer_config: if not tokenizer_config:
special_eos = special_last special_eos = special_last
elif special_last != special_eos:
if 'eot' not in self.special_token_types:
self.special_token_types = tuple(self.special_token_types) + ('eot', )
tokenizer_config['eot_token'] = special_eos
elif 'eom' not in self.special_token_types:
self.special_token_types = tuple(self.special_token_types) + ('eom', )
tokenizer_config['eom_token'] = special_eos
else:
logger.warning(f'Overriding EOS token {special_eos!r} with {special_last!r} without EOT/EOM fallback!')
tokenizer_config['eos_token'] = special_eos = special_last
self.add_special_token['eos'] = True if special_last == special_eos else False self.add_special_token['eos'] = True if special_last == special_eos else False
if special_last != special_eos: if special_last != special_eos:
logger.warning(f'Unknown trailing special token {special_last!r} in TemplateProcessing<single>') logger.warning(f'Unknown trailing special token {special_last!r} in TemplateProcessing<single>')