ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

llama : fix whitespace escaping in tokenizer (#2724)

Browse source
This commit is contained in:
goerch 2023-08-22 23:10:42 +02:00 • committed by GitHub
parent c63bb1d16a
commit 46ef5b5fcf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 16 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

13
tests/test-tokenizer-1.cpp
View file

@ -11,18 +11,11 @@
#include <locale>
static std::string escape_whitespace(const std::string& text) {
std::string result;
bool escaping = false;
result += "\xe2\x96\x81";
std::string result = "\xe2\x96\x81";
for (size_t offs = 0; offs < text.length(); ++offs) {
if (text[offs] == ' ') {
if (!escaping) {
result += "\xe2\x96\x81";
escaping = true;
}
}
else {
escaping = false;
result += "\xe2\x96\x81";
} else {
result += text[offs];
}
}