ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

imatrix : Add --parse-special for enabling parsing of special tokens in imatrix calculation (#13389)

Browse source 
* Add --parse-special for enabling parsing of special tokens in imatrix calculation

* whitespace
This commit is contained in:
Bartowski 2025-05-09 05:53:58 -04:00 • committed by GitHub
parent 0527771dd8
commit efb8b47eda
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
tools/imatrix/imatrix.cpp
View file

@ -24,7 +24,8 @@ static void print_usage(int, char ** argv) {
LOG("\n %s \\\n"
" -m model.gguf -f some-text.txt [-o imatrix.dat] [--process-output] \\\n"
" [--no-ppl] [--chunk 123] [--output-frequency 10] [--save-frequency 0] \\\n"
" [--in-file imatrix-prev-0.dat --in-file imatrix-prev-1.dat ...]\n" , argv[0]);
" [--in-file imatrix-prev-0.dat --in-file imatrix-prev-1.dat ...] \\\n"
" [--parse-special]\n" , argv[0]);
LOG("\n");
}
@ -439,7 +440,7 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params) {
auto tim1 = std::chrono::high_resolution_clock::now();
LOG_INF("%s: tokenizing the input ..\n", __func__);
std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, true);
std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, true, params.parse_special);
auto tim2 = std::chrono::high_resolution_clock::now();
LOG_INF("%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count());