ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

imatrix : Add --parse-special for enabling parsing of special tokens in imatrix calculation (#13389)

Browse source 
* Add --parse-special for enabling parsing of special tokens in imatrix calculation

* whitespace
This commit is contained in:
Bartowski 2025-05-09 05:53:58 -04:00 • committed by GitHub
parent 0527771dd8
commit efb8b47eda
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
common/arg.cpp
View file

@ -2627,6 +2627,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.i_chunk = value;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--parse-special"},
string_format("prase special tokens (chat, tool, etc) (default: %s)", params.parse_special ? "true" : "false"),
[](common_params & params) {
params.parse_special = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"-pps"},
string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),

1
common/common.h
View file

@ -409,6 +409,7 @@ struct common_params {
bool process_output = false; // collect data for the output tensor
bool compute_ppl = true; // whether to compute perplexity
bool parse_special = false; // whether to parse special tokens during imatrix tokenization
// cvector-generator params
int n_pca_batch = 100;