From 96f405393461062450692430e4916809bf71c3c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Podiv=C3=ADn?= <66251151+jpodivin@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:51:32 +0100 Subject: [PATCH] Adding logprobs to /v1/completions (#11344) Signed-off-by: Jiri Podivin --- examples/server/server.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 5f08c4ec..412908aa 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -267,6 +267,11 @@ struct server_task { params.speculative.n_min = std::max(params.speculative.n_min, 2); params.speculative.n_max = std::max(params.speculative.n_max, 0); + // Use OpenAI API logprobs only if n_probs wasn't provided + if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs){ + params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs); + } + if (data.contains("lora")) { if (data.at("lora").is_array()) { params.lora = parse_lora_request(params_base.lora_adapters, data.at("lora"));