diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 175473be80..8d2ea40cfe 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -16,6 +16,7 @@ src/test/modelconfig_test.cpp:656: OptionA src/test/modelconfig_test.cpp:662: OptionA src/test/modelconfig_test.cpp:668: OptionA src/test/modelinstance_test.cpp:1093: THROUGHTPUT +src/test/llm/output_parsers/qwen3_output_parser_test.cpp:720: thi third_party/aws-sdk-cpp/aws-sdk-cpp.bz WORKSPACE:98: thirdparty demos/classification_using_paddlepaddle_model/python/utils/imagenet_class_index.json @@ -38,6 +39,6 @@ windows_parse_tests.bat:136: SEH ==> SHE windows_parse_tests.bat:141: SEH ==> SHE windows_parse_tests.bat:144: SEH ==> SHE src/test/llm/output_parsers/gemma4_output_parser_test.cpp -src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this +src/test/llm/output_parsers/qwen3_output_parser_test.cpp:697: thi ==> the, this extras/chat_template_examples/chat_template_onyx.jinja src/test/llm/chat_templates/chat_template_onyx.jinja diff --git a/src/BUILD b/src/BUILD index 4fa62be9e2..1e2368327b 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2271,6 +2271,7 @@ cc_test( "//src/llm:genai_servables", "//src/llm:output_parsers", ":test_llm_output_parser_tests", + ":test_llm_io_processing_utils_tests", ":test_chat_template_workarounds", ":test_llm_input_processing_tests", ":test_llm_input_processing_integration_tests", @@ -2806,15 +2807,32 @@ cc_library( ) +cc_library( + name = "test_llm_io_processing_utils_tests", + linkstatic = 1, + alwayslink = True, + srcs = ["test/llm/io_processing_utils_test.cpp"], + deps = [ + "@com_google_googletest//:gtest", + "//src/llm:io_processing_utils", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, +) + cc_library( name = "test_llm_output_parser_tests", linkstatic = 1, alwayslink = True, + hdrs = ["test/llm/output_parsers/output_parser_test_utils.hpp"], srcs = glob(["test/llm/output_parsers/*_test.cpp"]), deps = [ "@com_google_googletest//:gtest", ":test_platform_utils", + "//src/llm:text_streamer", "//src/llm:output_parsers", + "//src/llm:io_processing_delta", + "//src/llm:openai_delta_serializer", ], copts = COPTS_TESTS, local_defines = COMMON_LOCAL_DEFINES, @@ -2824,6 +2842,7 @@ cc_library( name = "test_chat_template_workarounds", linkstatic = 1, alwayslink = True, + hdrs = ["test/llm/output_parsers/output_parser_test_utils.hpp"], srcs = [ "test/llm/chat_template_analyzer_test.cpp", "test/llm/chat_template_adapter_test.cpp", @@ -2836,6 +2855,8 @@ cc_library( "//src/llm:chat_template_probe", "//src/llm:io_processing_input_processors", "//src/llm:output_parsers", + "//src/llm:openai_delta_serializer", + "//src/llm:text_streamer", "//src/utils:env_guard", "//third_party:genai", ":test_platform_utils", diff --git a/src/llm/BUILD b/src/llm/BUILD index d4587ff119..599e30fa97 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -271,14 +271,39 @@ ovms_cc_library( ) ovms_cc_library( - name = "io_processing_base_output_parser", - hdrs = ["io_processing/base_output_parser.hpp"], - srcs = ["io_processing/base_output_parser.cpp"], + name = "io_processing_delta", + hdrs = [ + "io_processing/delta.hpp", + ], + deps = [], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "openai_delta_serializer", + hdrs = [ + "apis/openai_idelta_serializer.hpp", + "apis/openai_rapidjson_delta_serializer.hpp", + ], + srcs = ["apis/openai_rapidjson_delta_serializer.cpp"], deps = [ + ":io_processing_delta", "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_stringbuffer", "//src/port:rapidjson_writer", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_base_output_parser", + hdrs = ["io_processing/base_output_parser.hpp", + "io_processing/output_parsing_config.hpp"], + srcs = ["io_processing/base_output_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_document", + ":io_processing_delta", ":io_processing_utils", ":apis_tool_schema_wrapper", "//third_party:genai", @@ -312,9 +337,9 @@ ovms_cc_library( ) ovms_cc_library( - name = "io_processing_lfm2_utils", - hdrs = ["io_processing/lfm2/lfm2_utils.hpp"], - srcs = ["io_processing/lfm2/lfm2_utils.cpp"], + name = "io_processing_lfm2_tool_parser", + hdrs = ["io_processing/lfm2/lfm2_tool_parser.hpp"], + srcs = ["io_processing/lfm2/lfm2_tool_parser.cpp"], deps = [ "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_document", @@ -327,28 +352,6 @@ ovms_cc_library( visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "io_processing_lfm2_tool_parser", - hdrs = ["io_processing/lfm2/lfm2_tool_parser.hpp"], - srcs = ["io_processing/lfm2/lfm2_tool_parser.cpp"], - deps = [ - ":io_processing_lfm2_utils", - ], - visibility = ["//visibility:public"], -) - - -ovms_cc_library( - name = "io_processing_lfm25_tool_parser", - hdrs = ["io_processing/lfm2/lfm25_tool_parser.hpp"], - srcs = ["io_processing/lfm2/lfm25_tool_parser.cpp"], - deps = [ - ":io_processing_lfm2_utils", - "//src:libovmslogging", - ], - visibility = ["//visibility:public"], -) - ovms_cc_library( name = "io_processing_gemma4_tool_parser", hdrs = ["io_processing/gemma4/gemma4_tool_parser.hpp", "io_processing/gemma4/gemma4_reasoning_parser.hpp"], @@ -374,7 +377,6 @@ ovms_cc_library( ], srcs = [ "io_processing/minicpm5/minicpm5_tool_parser.cpp", - "io_processing/minicpm5/minicpm5_reasoning_parser.cpp", ], deps = [ "@com_github_tencent_rapidjson//:rapidjson", @@ -383,6 +385,7 @@ ovms_cc_library( "//src/utils:rapidjson_utils", ":io_processing_utils", ":io_processing_base_output_parser", + ":io_processing_qwen3_reasoning_parser", ":apis_tool_schema_wrapper", "//third_party:genai", ], @@ -405,6 +408,33 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_default_content_parser", + hdrs = ["io_processing/default_content_parser.hpp"], + srcs = ["io_processing/default_content_parser.cpp"], + deps = [ + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + "//src:libovmsstring_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_onyx_content_parser", + hdrs = ["io_processing/onyx/onyx_content_parser.hpp"], + srcs = ["io_processing/onyx/onyx_content_parser.cpp"], + deps = [ + "//src:libovmslogging", + "//src:libovmsstring_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "io_processing_onyx_tool_parser", hdrs = ["io_processing/onyx/onyx_tool_parser.hpp"], @@ -428,8 +458,6 @@ ovms_cc_library( hdrs = ["io_processing/onyx/onyx_reasoning_parser.hpp"], srcs = ["io_processing/onyx/onyx_reasoning_parser.cpp"], deps = [ - "@com_github_tencent_rapidjson//:rapidjson", - "//src/port:rapidjson_document", "//src:libovmslogging", ":io_processing_base_output_parser", "//third_party:genai", @@ -440,7 +468,6 @@ ovms_cc_library( ovms_cc_library( name = "io_processing_lfm25_reasoning_parser", hdrs = ["io_processing/lfm2/lfm25_reasoning_parser.hpp"], - srcs = ["io_processing/lfm2/lfm25_reasoning_parser.cpp"], deps = [ "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_document", @@ -448,6 +475,7 @@ ovms_cc_library( "//src:libovmsstring_utils", ":io_processing_utils", ":io_processing_base_output_parser", + ":io_processing_qwen3_reasoning_parser", "//third_party:genai", ], visibility = ["//visibility:public"], @@ -488,12 +516,13 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_parser_config_validation", ":io_processing_qwen3coder_tool_parser", ":io_processing_lfm2_tool_parser", - ":io_processing_lfm25_tool_parser", ":io_processing_gemma4_tool_parser", ":io_processing_minicpm5_tool_parser", ":io_processing_qwen3_reasoning_parser", ":io_processing_onyx_tool_parser", ":io_processing_onyx_reasoning_parser", + ":io_processing_onyx_content_parser", + ":io_processing_default_content_parser", ":io_processing_lfm25_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", @@ -521,10 +550,22 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "text_streamer", + hdrs = ["ovms_text_streamer.hpp"], + srcs = ["ovms_text_streamer.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src:libovmslogging", + "//third_party:genai", + ":output_parsers", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "genai_servables", hdrs = ["servable.hpp", - "ovms_text_streamer.hpp", "servable_initializer.hpp", "language_model/continuous_batching/servable.hpp", "language_model/continuous_batching/llm_executor.hpp", @@ -542,7 +583,6 @@ ovms_cc_library( "text_utils.hpp"], srcs = ["servable.cpp", "servable_initializer.cpp", - "ovms_text_streamer.cpp", "language_model/continuous_batching/servable.cpp", "language_model/continuous_batching/servable_initializer.cpp", "visual_language_model/continuous_batching/servable.cpp", @@ -569,6 +609,7 @@ ovms_cc_library( "//src/filesystem:libovmsfilesystem", "//src/tokenize:tokenize_parser", "llmcalculator_cc_proto", + ":text_streamer", ":openai_completions_api_handler", ":openai_responses_handler", ":generation_config_builders", diff --git a/src/llm/apis/openai_api_handler.cpp b/src/llm/apis/openai_api_handler.cpp index 4c998d67e8..6dddfd29db 100644 --- a/src/llm/apis/openai_api_handler.cpp +++ b/src/llm/apis/openai_api_handler.cpp @@ -99,19 +99,6 @@ std::string OpenAIApiHandler::serializeFailedEvent(const std::string& errorMessa return ""; } -std::vector OpenAIApiHandler::encodeTextToTokens(const std::string& text) { - auto result = tokenizer.encode(text); - auto& input_ids = result.input_ids; - if (input_ids.get_shape().size() != 2) - throw std::runtime_error("input_ids should have 2 dimensions"); - if (input_ids.get_shape()[0] != 1) - throw std::runtime_error("input_ids should have 1 batch size"); - if (input_ids.get_element_type() != ov::element::i64) - throw std::runtime_error("input_ids should have i64 element type"); - int64_t* data = reinterpret_cast(input_ids.data()); - return std::vector(data, data + input_ids.get_shape()[1]); -} - absl::Status OpenAIApiHandler::parseResponseFormat() { auto it = doc.FindMember("response_format"); if (it != doc.MemberEnd()) { @@ -270,6 +257,20 @@ absl::Status OpenAIApiHandler::parseTools() { return absl::OkStatus(); } +absl::Status OpenAIApiHandler::parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, + std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) { + auto status = parseRequestImpl(maxTokensLimit, bestOfLimit, maxModelLength, allowedLocalMediaPath, allowedMediaDomains); + if (status.ok()) + initOutputParser(); + return status; +} + +void OpenAIApiHandler::initOutputParser() { + if (toolParserName.empty() && reasoningParserName.empty()) + return; + outputParser = std::make_shared(tokenizer, toolParserName, reasoningParserName, request.toolNameSchemaMap); +} + absl::StatusOr> OpenAIApiHandler::parseToolsToJsonContainer() { auto it = doc.FindMember("tools"); if (it == doc.MemberEnd() || it->value.IsNull()) { @@ -395,15 +396,37 @@ void OpenAIApiHandler::incrementProcessedTokens(size_t numTokens) { usage.completionTokens += numTokens; } -ParsedOutput OpenAIApiHandler::parseOutputIfNeeded(const std::vector& generatedIds) { - OVMS_PROFILE_FUNCTION(); - ParsedOutput parsedOutput; - if ((endpoint != Endpoint::CHAT_COMPLETIONS && endpoint != Endpoint::RESPONSES) || outputParser == nullptr) { - parsedOutput.content = this->tokenizer.decode(generatedIds, ov::genai::skip_special_tokens(request.skipSpecialTokens)); - } else { - parsedOutput = outputParser->parse(generatedIds, this->areToolsAvailable()); - } - return parsedOutput; +std::string OpenAIApiHandler::serializeUnaryResponse( + const std::vector>& allDeltas, + const std::vector& finishReasons) { + return serializeUnaryResponse(allDeltas, finishReasons, {}); +} + +ParsedOutput OpenAIApiHandler::parsedOutputFromDeltas(const std::vector& deltas) { + ParsedOutput output; + std::vector toolCalls; + for (const Delta& d : deltas) { + std::visit(overloaded{ + [&](const ContentDelta& x) { output.content += x.text; }, + [&](const ReasoningDelta& x) { output.reasoning += x.text; }, + [&](const ToolCallDelta& x) { + const auto idx = static_cast(x.index); + if (idx >= toolCalls.size()) + toolCalls.resize(idx + 1); + ToolCall& tc = toolCalls[idx]; + if (x.id) + tc.id = *x.id; + if (x.name) + tc.name = *x.name; + tc.arguments += x.arguments; + }, + [&](const FinishDelta&) {}, + [&](const AudioDelta&) {}, + }, + d); + } + output.toolCalls = std::move(toolCalls); + return output; } // --- Free functions --- @@ -756,9 +779,6 @@ absl::Status OpenAIApiHandler::parseCommonPart(std::optional maxTokens return absl::InvalidArgumentError("skip_special_tokens is not a bool"); request.skipSpecialTokens = it->value.GetBool(); } - if (!request.skipSpecialTokens && outputParser != nullptr) { - outputParser.reset(); - } request.maxModelLength = maxModelLength; diff --git a/src/llm/apis/openai_api_handler.hpp b/src/llm/apis/openai_api_handler.hpp index 85fd0ad7a3..0f3c0b3d64 100644 --- a/src/llm/apis/openai_api_handler.hpp +++ b/src/llm/apis/openai_api_handler.hpp @@ -87,6 +87,13 @@ struct CompletionUsageStatistics { } }; +// Per-choice raw token data needed to build logprob objects in unary responses. +// populated in GenAiServable::prepareCompleteResponse from GenerationOutput. +struct UnaryChoiceLogprobs { + std::vector generatedIds; + std::vector logProbs; +}; + // Abstract base class for OpenAI API handlers. // Holds common state (request, doc, tokenizer, usage, output parser) and implements // shared parsing logic. Endpoint-specific parsing and serialization are pure virtual. @@ -98,6 +105,8 @@ class OpenAIApiHandler { OpenAIRequest request; std::chrono::time_point created; ov::genai::Tokenizer tokenizer; + const std::string toolParserName; + const std::string reasoningParserName; // Output parser is used to parse chat completions response to extract specific fields like tool calls and reasoning. std::shared_ptr outputParser = nullptr; @@ -113,14 +122,16 @@ class OpenAIApiHandler { std::vector verboseRawTokens; std::string verboseRawText; + virtual absl::Status parseRequestImpl(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, + std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) = 0; + // Shared parsing helpers absl::Status parseCommonPart(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength); absl::Status parseResponseFormat(); absl::Status ensureArgumentsInToolCalls(Value& messageObj); - ParsedOutput parseOutputIfNeeded(const std::vector& generatedIds); - - // Shared VLM workaround: encode text to tokens using tokenizer, validates shape - std::vector encodeTextToTokens(const std::string& text); + void initOutputParser(); + // Assemble a ParsedOutput from a sequence of streaming Delta variants produced by OVMSTextStreamer. + static ParsedOutput parsedOutputFromDeltas(const std::vector& deltas); public: OpenAIApiHandler(Document& doc, Endpoint endpoint, std::chrono::time_point creationTime, @@ -128,13 +139,9 @@ class OpenAIApiHandler { doc(doc), endpoint(endpoint), created(creationTime), - tokenizer(tokenizer) { - // TODO we should delay creating output parser until we have request with toolNameSchemaMap parsed - // we pass it now, but it has to be populated first before first use - if (!toolParserName.empty() || !reasoningParserName.empty()) { - outputParser = std::make_shared(tokenizer, toolParserName, reasoningParserName, this->request.toolNameSchemaMap); - } - } + tokenizer(tokenizer), + toolParserName(toolParserName), + reasoningParserName(reasoningParserName) {} virtual ~OpenAIApiHandler() = default; @@ -144,9 +151,9 @@ class OpenAIApiHandler { OpenAIApiHandler(OpenAIApiHandler&&) = delete; OpenAIApiHandler& operator=(OpenAIApiHandler&&) = delete; - // Request parsing - pure virtual, each handler implements its own endpoint-specific dispatch - virtual absl::Status parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, - std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt) = 0; + // Request parsing: non-virtual wrapper; calls parseRequestImpl() then initOutputParser(). + absl::Status parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, + std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt); // Shared parsing (non-virtual) absl::Status parseTools(); @@ -198,10 +205,20 @@ class OpenAIApiHandler { virtual void incrementProcessedTokens(size_t numTokens = 1); // Serialization - pure virtual, each handler produces its own response format - virtual std::string serializeUnaryResponse(const std::vector& generationOutputs) = 0; - virtual std::string serializeUnaryResponse(ov::genai::EncodedResults& results) = 0; - virtual std::string serializeUnaryResponse(ov::genai::VLMDecodedResults& results, const std::string& textResponse) = 0; - virtual std::string serializeStreamingChunk(rapidjson::Document parsedDelta, ov::genai::GenerationFinishReason finishReason) = 0; + // Delta-based unary serialisation — assembles a complete response from streaming + // deltas collected via deltaChannel after OVMSTextStreamer finishes. + // Single-choice variant (used by Legacy servables). + virtual std::string serializeUnaryResponse(const std::vector& deltas, ov::genai::GenerationFinishReason finishReason) = 0; + // Multi-choice variant: N delta-vectors (one per sequence) + per-sequence finish reasons. + // logprobData may be empty when logprobs are not requested; otherwise its size equals + // allDeltas.size(). Used by ContinuousBatchingServable for both n=1 and n>1. + virtual std::string serializeUnaryResponse(const std::vector>& allDeltas, + const std::vector& finishReasons, + const std::vector& logprobData) = 0; + // Convenience overload: no logprobs (delegates to the virtual above with empty logprobData). + std::string serializeUnaryResponse(const std::vector>& allDeltas, + const std::vector& finishReasons); + virtual std::string serializeStreamingChunk(Delta delta, ov::genai::GenerationFinishReason finishReason) = 0; virtual std::string serializeStreamingUsageChunk() = 0; virtual std::string serializeStreamingHandshakeChunk() = 0; diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp index e2aa02d8ee..40e32ee9ee 100644 --- a/src/llm/apis/openai_completions.cpp +++ b/src/llm/apis/openai_completions.cpp @@ -46,17 +46,50 @@ using namespace rapidjson; namespace ovms { -static bool hasToolCallsInStreamingDelta(const rapidjson::Document& delta) { - if (!delta.HasMember("delta") || !delta["delta"].IsObject()) { - return false; - } - const auto& deltaObj = delta["delta"]; - return deltaObj.HasMember("tool_calls") && deltaObj["tool_calls"].IsArray(); +static bool hasToolCallsInStreamingDelta(const Delta& delta) { + return std::holds_alternative(delta); +} + +Value OpenAIChatCompletionsHandler::serializeDeltaValue(const Delta& delta, Document::AllocatorType& allocator) { + return std::visit(overloaded{ + [&](const ContentDelta& d) -> Value { + Value v(kObjectType); + v.AddMember("content", Value(d.text.c_str(), allocator), allocator); + return v; + }, + [&](const ReasoningDelta& d) -> Value { + Value v(kObjectType); + v.AddMember("reasoning_content", Value(d.text.c_str(), allocator), allocator); + return v; + }, + [&](const ToolCallDelta& d) -> Value { + Value tcObj(kObjectType); + if (d.id) { + tcObj.AddMember("id", Value(d.id->c_str(), allocator), allocator); + tcObj.AddMember("type", Value("function", allocator), allocator); + } + tcObj.AddMember("index", d.index, allocator); + Value fn(kObjectType); + if (d.name) + fn.AddMember("name", Value(d.name->c_str(), allocator), allocator); + if (!d.arguments.empty()) + fn.AddMember("arguments", Value(d.arguments.c_str(), allocator), allocator); + tcObj.AddMember("function", fn, allocator); + Value arr(kArrayType); + arr.PushBack(tcObj, allocator); + Value v(kObjectType); + v.AddMember("tool_calls", arr, allocator); + return v; + }, + [&](const FinishDelta&) -> Value { return Value(kObjectType); }, + [&](const AudioDelta&) -> Value { return Value(kObjectType); }, + }, + delta); } // --- Request parsing --- -absl::Status OpenAIChatCompletionsHandler::parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, +absl::Status OpenAIChatCompletionsHandler::parseRequestImpl(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) { absl::Status status = parseCommonPart(maxTokensLimit, bestOfLimit, maxModelLength); if (status != absl::OkStatus()) @@ -256,185 +289,38 @@ absl::Status OpenAIChatCompletionsHandler::parseMessages(std::optional& generationOutputs) { +std::string OpenAIChatCompletionsHandler::serializeUnaryResponse( + const std::vector& deltas, + ov::genai::GenerationFinishReason finishReason) { OVMS_PROFILE_FUNCTION(); + ParsedOutput parsedOutput = parsedOutputFromDeltas(deltas); OpenAiJsonResponse jsonResponse; jsonResponse.StartObject(); - // choices: array of size N, where N is related to n request parameter jsonResponse.StartArray("choices"); - int index = 0; - // Manual usage setup for CB pipelines. For legacy we rely on PerfMetrics object from GenAI `generate` results - usage.completionTokens = 0; - for (const ov::genai::GenerationOutput& generationOutput : generationOutputs) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Generated tokens: {}", generationOutput.generated_ids); - - updateUsage(usage, generationOutput.generated_ids, request.echo); - ParsedOutput parsedOutput = parseOutputIfNeeded(generationOutput.generated_ids); - - jsonResponse.StartObject(); - // finish_reason: string; - // "stop" => natural stop point due to stopping criteria - // "length" => due to reaching max_tokens parameter - // "tool_calls" => generation stopped due to generated tool calls - - std::optional finishReason = mapFinishReason(generationOutput.finish_reason, !parsedOutput.toolCalls.empty()); - if (!finishReason.has_value()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unknown finish reason: {}", static_cast(generationOutput.finish_reason)); - } - jsonResponse.FinishReason(finishReason.value_or("unknown")); - // index: integer; Choice index, only n=1 supported anyway - jsonResponse.Index(index++); - - // TODO: logprobs: object/null; Log probability information for the choice. - if (this->request.logprobschat || this->request.logprobs) { - jsonResponse.StartObject("logprobs"); - if (endpoint == Endpoint::CHAT_COMPLETIONS) { - jsonResponse.StartArray("content"); - - for (int i = 0; i < generationOutput.generated_ids.size(); i++) { - std::string token = tokenizer.decode(std::vector({generationOutput.generated_ids[i]}), ov::genai::skip_special_tokens(this->request.skipSpecialTokens)); - float logprob = generationOutput.generated_log_probs[i]; - jsonResponse.LogprobObject(token, logprob); - } - jsonResponse.EndArray(); - } - if (endpoint == Endpoint::COMPLETIONS) { - jsonResponse.StartArray("tokens"); - for (int i = 0; i < generationOutput.generated_ids.size(); i++) { - std::string token = tokenizer.decode(std::vector({generationOutput.generated_ids[i]}), ov::genai::skip_special_tokens(this->request.skipSpecialTokens)); - jsonResponse.String(token); - } - jsonResponse.EndArray(); - - jsonResponse.StartArray("token_logprobs"); - for (int i = 0; i < generationOutput.generated_ids.size(); i++) { - float logprob = generationOutput.generated_log_probs[i]; - jsonResponse.LogprobValue(logprob); - } - jsonResponse.EndArray(); - - jsonResponse.StartArray("top_logprobs"); - for (int i = 0; i < generationOutput.generated_ids.size(); i++) { - jsonResponse.StartObject(); - std::string token = tokenizer.decode(std::vector({generationOutput.generated_ids[i]}), ov::genai::skip_special_tokens(this->request.skipSpecialTokens)); - float logprob = generationOutput.generated_log_probs[i]; - jsonResponse.Logprob(token, logprob); - jsonResponse.EndObject(); - } - jsonResponse.EndArray(); - - jsonResponse.StartArray("text_offset"); - for (int i = 0; i < generationOutput.generated_ids.size(); i++) { - if (i == 0) { - jsonResponse.TextOffsetValue(0); - } else { - std::string textBeforeToken = tokenizer.decode(std::vector({generationOutput.generated_ids.begin(), generationOutput.generated_ids.begin() + i}), ov::genai::skip_special_tokens(this->request.skipSpecialTokens)); - jsonResponse.TextOffsetValue(textBeforeToken.size()); - } - } - jsonResponse.EndArray(); - } - jsonResponse.EndObject(); - } else { - jsonResponse.Null("logprobs"); // "logprobs": null - } - - if (endpoint == Endpoint::CHAT_COMPLETIONS) { - jsonResponse.MessageObject(parsedOutput); - } else if (endpoint == Endpoint::COMPLETIONS) { - jsonResponse.Text(parsedOutput); - } + jsonResponse.StartObject(); - // finish message object - jsonResponse.EndObject(); + auto finishReasonStr = mapFinishReason(finishReason, !parsedOutput.toolCalls.empty()); + if (!finishReasonStr.has_value()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unknown finish reason: {}", static_cast(finishReason)); } - // finish choices array - jsonResponse.EndArray(); - - // created: integer; Unix timestamp (in seconds) when the MP graph was created. - jsonResponse.Int("created", std::chrono::duration_cast(created.time_since_epoch()).count()); + jsonResponse.FinishReason(finishReasonStr.value_or("unknown")); + jsonResponse.Index(0); + jsonResponse.Null("logprobs"); - // model: string; copied from the request - jsonResponse.String("model", request.model); - - // object: string; defined that the type is unary rather than streamed chunk if (endpoint == Endpoint::CHAT_COMPLETIONS) { - jsonResponse.String("object", "chat.completion"); + jsonResponse.MessageObject(parsedOutput); } else if (endpoint == Endpoint::COMPLETIONS) { - jsonResponse.String("object", "text_completion"); - } - - jsonResponse.UsageObject(usage); - - // TODO: id: string; A unique identifier for the chat completion. - - // TODO: system_fingerprint: string; This fingerprint represents the backend configuration that the model runs with. - // Can be used in conjunction with the seed request parameter to understand when backend changes have been made that might impact determinism. - - if (isVerboseResponse()) { - jsonResponse.StartObject("__verbose"); - jsonResponse.String("prompt", getVerbosePrompt()); - std::string rawContent; - if (!generationOutputs.empty()) { - rawContent = tokenizer.decode(generationOutputs.front().generated_ids, ov::genai::skip_special_tokens(false)); - } - jsonResponse.String("content", rawContent); - jsonResponse.EndObject(); + jsonResponse.Text(parsedOutput); } - // finish response object jsonResponse.EndObject(); - return jsonResponse.ToString(); -} - -std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::EncodedResults& results) { - OVMS_PROFILE_FUNCTION(); - usage.promptTokens = results.perf_metrics.get_num_input_tokens(); - usage.completionTokens = results.perf_metrics.get_num_generated_tokens(); - - OpenAiJsonResponse jsonResponse; - jsonResponse.StartObject(); - - // choices: array of size N, where N is related to n request parameter - jsonResponse.StartArray("choices"); - if (results.finish_reasons.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary LM generation result, defaulting to STOP for all choices"); - } else if (results.finish_reasons.size() != results.tokens.size()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Finish reasons size ({}) does not match tokens size ({}) in unary LM generation result, defaulting missing entries to STOP", - results.finish_reasons.size(), results.tokens.size()); - } - for (size_t i = 0; i < results.tokens.size(); ++i) { - const std::vector& tokens = results.tokens[i]; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Generated tokens: {}", tokens); - ParsedOutput parsedOutput = parseOutputIfNeeded(tokens); - jsonResponse.StartObject(); - const ov::genai::GenerationFinishReason finishReasonRaw = i < results.finish_reasons.size() ? results.finish_reasons[i] : ov::genai::GenerationFinishReason::STOP; - auto finishReason = mapFinishReason(finishReasonRaw, !parsedOutput.toolCalls.empty()); - jsonResponse.FinishReason(finishReason.value_or("unknown")); - // index: integer; Choice index, only n=1 supported anyway - jsonResponse.Index(static_cast(i)); - - if (endpoint == Endpoint::CHAT_COMPLETIONS) { - jsonResponse.MessageObject(parsedOutput); - } else if (endpoint == Endpoint::COMPLETIONS) { - jsonResponse.Text(parsedOutput); - } - - // finish message object - jsonResponse.EndObject(); - } - // finish choices array jsonResponse.EndArray(); - // created: integer; Unix timestamp (in seconds) when the MP graph was created. jsonResponse.Int("created", std::chrono::duration_cast(created.time_since_epoch()).count()); - - // model: string; copied from the request jsonResponse.String("model", request.model); - // object: string; defined that the type is unary rather than streamed chunk if (endpoint == Endpoint::CHAT_COMPLETIONS) { jsonResponse.String("object", "chat.completion"); } else if (endpoint == Endpoint::COMPLETIONS) { @@ -443,59 +329,95 @@ std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::Enco jsonResponse.UsageObject(usage); - // TODO: id: string; A unique identifier for the chat completion. - - // TODO: system_fingerprint: string; This fingerprint represents the backend configuration that the model runs with. - // Can be used in conjunction with the seed request parameter to understand when backend changes have been made that might impact determinism. - if (isVerboseResponse()) { jsonResponse.StartObject("__verbose"); jsonResponse.String("prompt", getVerbosePrompt()); - std::string rawContent; - if (!results.tokens.empty()) { - rawContent = tokenizer.decode(results.tokens.front(), ov::genai::skip_special_tokens(false)); - } - jsonResponse.String("content", rawContent); + jsonResponse.String("content", getVerboseRawText()); jsonResponse.EndObject(); } - // finish response object jsonResponse.EndObject(); return jsonResponse.ToString(); } -std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::VLMDecodedResults& results, const std::string& textResponse) { +std::string OpenAIChatCompletionsHandler::serializeUnaryResponse( + const std::vector>& allDeltas, + const std::vector& finishReasons, + const std::vector& logprobData) { OVMS_PROFILE_FUNCTION(); - usage.promptTokens = results.perf_metrics.get_num_input_tokens(); - usage.completionTokens = results.perf_metrics.get_num_generated_tokens(); OpenAiJsonResponse jsonResponse; jsonResponse.StartObject(); - // choices: array of size N, where N is related to n request parameter jsonResponse.StartArray("choices"); - int index = 0; + for (size_t i = 0; i < allDeltas.size(); ++i) { + ParsedOutput parsedOutput = parsedOutputFromDeltas(allDeltas[i]); - if (!textResponse.empty()) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Generated text: {}", textResponse); + jsonResponse.StartObject(); - // Workaround to use OVMS unary parsers: get tokens from string - // This way we have detokenized text from GenAI and calculate tokens, to further convert back to text again, in parseOutputIfNeeded... - auto generatedTokens = encodeTextToTokens(textResponse); + const ov::genai::GenerationFinishReason finishReason = + (i < finishReasons.size()) ? finishReasons[i] : ov::genai::GenerationFinishReason::STOP; + auto finishReasonStr = mapFinishReason(finishReason, !parsedOutput.toolCalls.empty()); + if (!finishReasonStr.has_value()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unknown finish reason: {}", static_cast(finishReason)); + } + jsonResponse.FinishReason(finishReasonStr.value_or("unknown")); + jsonResponse.Index(static_cast(i)); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Generated tokens: {}", generatedTokens); - ParsedOutput parsedOutput = parseOutputIfNeeded(generatedTokens); - jsonResponse.StartObject(); - if (results.finish_reasons.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary VLM generation result, defaulting to STOP"); + const bool hasChoiceLogprobs = !logprobData.empty() && + i < logprobData.size() && + !logprobData[i].generatedIds.empty() && + (request.logprobschat || request.logprobs); + if (hasChoiceLogprobs) { + jsonResponse.StartObject("logprobs"); + if (endpoint == Endpoint::CHAT_COMPLETIONS) { + jsonResponse.StartArray("content"); + for (size_t j = 0; j < logprobData[i].generatedIds.size(); ++j) { + std::string token = tokenizer.decode(std::vector({logprobData[i].generatedIds[j]}), + ov::genai::skip_special_tokens(request.skipSpecialTokens)); + const float logprob = (j < logprobData[i].logProbs.size()) ? logprobData[i].logProbs[j] : 0.0f; + jsonResponse.LogprobObject(token, logprob); + } + jsonResponse.EndArray(); + } + if (endpoint == Endpoint::COMPLETIONS) { + jsonResponse.StartArray("tokens"); + for (size_t j = 0; j < logprobData[i].generatedIds.size(); ++j) { + jsonResponse.String(tokenizer.decode(std::vector({logprobData[i].generatedIds[j]}), + ov::genai::skip_special_tokens(request.skipSpecialTokens))); + } + jsonResponse.EndArray(); + + jsonResponse.StartArray("token_logprobs"); + for (size_t j = 0; j < logprobData[i].generatedIds.size(); ++j) { + jsonResponse.LogprobValue((j < logprobData[i].logProbs.size()) ? logprobData[i].logProbs[j] : 0.0f); + } + jsonResponse.EndArray(); + + jsonResponse.StartArray("top_logprobs"); + for (size_t j = 0; j < logprobData[i].generatedIds.size(); ++j) { + jsonResponse.StartObject(); + const std::string token = tokenizer.decode(std::vector({logprobData[i].generatedIds[j]}), + ov::genai::skip_special_tokens(request.skipSpecialTokens)); + jsonResponse.Logprob(token, (j < logprobData[i].logProbs.size()) ? logprobData[i].logProbs[j] : 0.0f); + jsonResponse.EndObject(); + } + jsonResponse.EndArray(); + + jsonResponse.StartArray("text_offset"); + size_t offset = 0; + for (size_t j = 0; j < logprobData[i].generatedIds.size(); ++j) { + jsonResponse.TextOffsetValue(static_cast(offset)); + offset += tokenizer.decode(std::vector({logprobData[i].generatedIds[j]}), + ov::genai::skip_special_tokens(request.skipSpecialTokens)) + .size(); + } + jsonResponse.EndArray(); + } + jsonResponse.EndObject(); + } else { + jsonResponse.Null("logprobs"); } - // Current generation flow uses batch=1, so only finish_reasons[0] is expected here. - const ov::genai::GenerationFinishReason finishReasonRaw = results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : results.finish_reasons[0]; - auto finishReason = mapFinishReason(finishReasonRaw, !parsedOutput.toolCalls.empty()); - jsonResponse.FinishReason(finishReason.value_or("unknown")); - // index: integer; Choice index, only n=1 supported anyway - jsonResponse.Index(index++); - // TODO: logprobs: object/null; Log probability information for the choice. if (endpoint == Endpoint::CHAT_COMPLETIONS) { jsonResponse.MessageObject(parsedOutput); @@ -503,19 +425,13 @@ std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::VLMD jsonResponse.Text(parsedOutput); } - // finish message object jsonResponse.EndObject(); } - // finish choices array jsonResponse.EndArray(); - // created: integer; Unix timestamp (in seconds) when the MP graph was created. jsonResponse.Int("created", std::chrono::duration_cast(created.time_since_epoch()).count()); - - // model: string; copied from the request jsonResponse.String("model", request.model); - // object: string; defined that the type is unary rather than streamed chunk if (endpoint == Endpoint::CHAT_COMPLETIONS) { jsonResponse.String("object", "chat.completion"); } else if (endpoint == Endpoint::COMPLETIONS) { @@ -524,27 +440,20 @@ std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::VLMD jsonResponse.UsageObject(usage); - // TODO: id: string; A unique identifier for the chat completion. - - // TODO: system_fingerprint: string; This fingerprint represents the backend configuration that the model runs with. - // Can be used in conjunction with the seed request parameter to understand when backend changes have been made that might impact determinism. - if (isVerboseResponse()) { jsonResponse.StartObject("__verbose"); jsonResponse.String("prompt", getVerbosePrompt()); - // For VLM the raw decoded text is provided by GenAI directly. - jsonResponse.String("content", textResponse); + jsonResponse.String("content", getVerboseRawText()); jsonResponse.EndObject(); } - // finish response object jsonResponse.EndObject(); return jsonResponse.ToString(); } // --- Streaming serialization --- -std::string OpenAIChatCompletionsHandler::serializeStreamingChunk(rapidjson::Document parsedDelta, ov::genai::GenerationFinishReason finishReason) { +std::string OpenAIChatCompletionsHandler::serializeStreamingChunk(Delta delta, ov::genai::GenerationFinishReason finishReason) { OVMS_PROFILE_FUNCTION(); Document doc; @@ -570,26 +479,15 @@ std::string OpenAIChatCompletionsHandler::serializeStreamingChunk(rapidjson::Doc // TODO: logprobs: object/null; Log probability information for the choice. choice.AddMember("logprobs", Value(), allocator); if (endpoint == Endpoint::CHAT_COMPLETIONS) { - // parsedDelta is a pre-parsed Document produced by OVMSTextStreamer::flush_chunk. - // Shape: {"delta":{...}} for content/reasoning/tool_calls, or an empty Document{} - // for finish-only chunks (generation ended on a swallowed token). - if (parsedDelta.HasMember("delta")) { - choice.AddMember("delta", Value(parsedDelta["delta"], allocator), allocator); - hasToolCalls = hasToolCallsInStreamingDelta(parsedDelta); - if (hasToolCalls) { - toolCallsDetectedInStream = true; - } - } else { - // No delta from the parser (e.g. generation ended on a swallowed token). - // The OpenAI API requires "delta" to always be present in each choice, so emit an empty object. - Value emptyDelta(kObjectType); - choice.AddMember("delta", emptyDelta, allocator); - } + hasToolCalls = hasToolCallsInStreamingDelta(delta); + if (hasToolCalls) + toolCallsDetectedInStream = true; + Value deltaVal = serializeDeltaValue(delta, allocator); + choice.AddMember("delta", deltaVal, allocator); } else if (endpoint == Endpoint::COMPLETIONS) { - // For /v1/completions, extract the plain text from the content delta. - if (parsedDelta.HasMember("delta") && parsedDelta["delta"].IsObject() && - parsedDelta["delta"].HasMember("content") && parsedDelta["delta"]["content"].IsString()) { - choice.AddMember("text", Value(parsedDelta["delta"]["content"].GetString(), allocator), allocator); + // For /v1/completions extract plain text from ContentDelta only. + if (const auto* cd = std::get_if(&delta)) { + choice.AddMember("text", Value(cd->text.c_str(), allocator), allocator); } else { choice.AddMember("text", Value("", allocator), allocator); } @@ -742,8 +640,23 @@ std::string OpenAIChatCompletionsHandler::serializeStreamingHandshakeChunk() { } void OpenAIChatCompletionsHandler::incrementProcessedTokens(size_t numTokens) { + const size_t previousProcessed = processedTokens; processedTokens += numTokens; - if (!request.echo || processedTokens > usage.promptTokens) + + if (!request.echo) { usage.completionTokens += numTokens; + return; + } + + // Echo mode may deliver prompt+completion in one unary batch. Count only + // the incremental portion that lies beyond prompt_tokens. + const size_t previousCompletionBoundary = + (previousProcessed > usage.promptTokens) ? (previousProcessed - usage.promptTokens) : 0; + const size_t currentCompletionBoundary = + (processedTokens > usage.promptTokens) ? (processedTokens - usage.promptTokens) : 0; + + if (currentCompletionBoundary > previousCompletionBoundary) { + usage.completionTokens += (currentCompletionBoundary - previousCompletionBoundary); + } } } // namespace ovms diff --git a/src/llm/apis/openai_completions.hpp b/src/llm/apis/openai_completions.hpp index 7b1059fb6c..af69a611f8 100644 --- a/src/llm/apis/openai_completions.hpp +++ b/src/llm/apis/openai_completions.hpp @@ -30,17 +30,21 @@ class OpenAIChatCompletionsHandler : public OpenAIApiHandler { absl::Status parseCompletionsPart(); absl::Status parseChatCompletionsPart(std::optional maxTokensLimit, std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains); + // Builds the "delta" object of a streaming chat-completions chunk from a typed Delta variant. + static Value serializeDeltaValue(const Delta& delta, Document::AllocatorType& allocator); + public: using OpenAIApiHandler::OpenAIApiHandler; // Inherit constructors - absl::Status parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, - std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt) override; + absl::Status parseRequestImpl(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, + std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) override; absl::Status parseMessages(std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt); - std::string serializeUnaryResponse(const std::vector& generationOutputs) override; - std::string serializeUnaryResponse(ov::genai::EncodedResults& results) override; - std::string serializeUnaryResponse(ov::genai::VLMDecodedResults& results, const std::string& textResponse) override; - std::string serializeStreamingChunk(rapidjson::Document parsedDelta, ov::genai::GenerationFinishReason finishReason) override; + std::string serializeUnaryResponse(const std::vector& deltas, ov::genai::GenerationFinishReason finishReason) override; + std::string serializeUnaryResponse(const std::vector>& allDeltas, + const std::vector& finishReasons, + const std::vector& logprobData) override; + std::string serializeStreamingChunk(Delta delta, ov::genai::GenerationFinishReason finishReason) override; std::string serializeStreamingUsageChunk() override; std::string serializeStreamingHandshakeChunk() override; void incrementProcessedTokens(size_t numTokens = 1) override; diff --git a/src/llm/apis/openai_idelta_serializer.hpp b/src/llm/apis/openai_idelta_serializer.hpp new file mode 100644 index 0000000000..dc1d160c9f --- /dev/null +++ b/src/llm/apis/openai_idelta_serializer.hpp @@ -0,0 +1,36 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +#include "src/llm/io_processing/delta.hpp" + +namespace ovms { +class IDeltaSerializer { +public: + virtual ~IDeltaSerializer() = default; + + virtual std::string serialize(const ContentDelta& delta) const = 0; + virtual std::string serialize(const ReasoningDelta& delta) const = 0; + virtual std::string serialize(const ToolCallDelta& delta) const = 0; + virtual std::string serialize(const FinishDelta& delta) const = 0; + virtual std::string serialize(const AudioDelta& delta) const = 0; + + std::string serialize(const Delta& delta) const { + return std::visit([this](const auto& d) { return serialize(d); }, delta); + } +}; + +} // namespace ovms diff --git a/src/llm/apis/openai_rapidjson_delta_serializer.cpp b/src/llm/apis/openai_rapidjson_delta_serializer.cpp new file mode 100644 index 0000000000..05c34b1ab0 --- /dev/null +++ b/src/llm/apis/openai_rapidjson_delta_serializer.cpp @@ -0,0 +1,93 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/port/rapidjson_stringbuffer.hpp" +#include "src/port/rapidjson_writer.hpp" + +#include "src/llm/apis/openai_rapidjson_delta_serializer.hpp" + +namespace ovms { + +std::string RapidJsonDeltaSerializer::serialize(const ContentDelta& d) const { + rapidjson::StringBuffer buf; + rapidjson::Writer w(buf); + w.StartObject(); + w.Key("delta"); + w.StartObject(); + w.Key("content"); + w.String(d.text.c_str(), static_cast(d.text.size())); + w.EndObject(); + w.EndObject(); + return buf.GetString(); +} + +std::string RapidJsonDeltaSerializer::serialize(const ReasoningDelta& d) const { + rapidjson::StringBuffer buf; + rapidjson::Writer w(buf); + w.StartObject(); + w.Key("delta"); + w.StartObject(); + w.Key("reasoning_content"); + w.String(d.text.c_str(), static_cast(d.text.size())); + w.EndObject(); + w.EndObject(); + return buf.GetString(); +} + +std::string RapidJsonDeltaSerializer::serialize(const ToolCallDelta& d) const { + rapidjson::StringBuffer buf; + rapidjson::Writer w(buf); + w.StartObject(); + w.Key("delta"); + w.StartObject(); + w.Key("tool_calls"); + w.StartArray(); + w.StartObject(); + if (d.id) { + w.Key("id"); + w.String(d.id->c_str(), static_cast(d.id->size())); + w.Key("type"); + w.String("function"); + } + w.Key("index"); + w.Int(d.index); + w.Key("function"); + w.StartObject(); + if (d.name) { + w.Key("name"); + w.String(d.name->c_str(), static_cast(d.name->size())); + } + if (!d.arguments.empty()) { + w.Key("arguments"); + w.String(d.arguments.c_str(), static_cast(d.arguments.size())); + } + w.EndObject(); + w.EndObject(); + w.EndArray(); + w.EndObject(); + w.EndObject(); + return buf.GetString(); +} + +std::string RapidJsonDeltaSerializer::serialize(const FinishDelta&) const { + return "{}"; +} + +std::string RapidJsonDeltaSerializer::serialize(const AudioDelta&) const { + return "{}"; +} + +} // namespace ovms diff --git a/src/llm/apis/openai_rapidjson_delta_serializer.hpp b/src/llm/apis/openai_rapidjson_delta_serializer.hpp new file mode 100644 index 0000000000..8ccf70e19d --- /dev/null +++ b/src/llm/apis/openai_rapidjson_delta_serializer.hpp @@ -0,0 +1,43 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +#include "src/llm/io_processing/delta.hpp" +#include "src/llm/apis/openai_idelta_serializer.hpp" + +namespace ovms { + +// Serializes Delta variants to the OpenAI streaming delta JSON schema using rapidjson. +// +// Output shapes: +// ContentDelta → {"delta":{"content":""}} +// ReasoningDelta → {"delta":{"reasoning_content":""}} +// ToolCallDelta → first delta (id/name present): +// {"delta":{"tool_calls":[{"id":"","type":"function","index":,"function":{"name":""}}]}} +// argument delta (id/name nullopt): +// {"delta":{"tool_calls":[{"index":,"function":{"arguments":""}}]}} +// FinishDelta → {} +// AudioDelta → {} +class RapidJsonDeltaSerializer : public IDeltaSerializer { +public: + std::string serialize(const ContentDelta& delta) const override; + std::string serialize(const ReasoningDelta& delta) const override; + std::string serialize(const ToolCallDelta& delta) const override; + std::string serialize(const FinishDelta& delta) const override; + std::string serialize(const AudioDelta& delta) const override; +}; + +} // namespace ovms diff --git a/src/llm/apis/openai_responses.cpp b/src/llm/apis/openai_responses.cpp index f5f0ab779c..3e7ef45d22 100644 --- a/src/llm/apis/openai_responses.cpp +++ b/src/llm/apis/openai_responses.cpp @@ -576,7 +576,7 @@ class ChatHistorySink { // --- Request parsing --- -absl::Status OpenAIResponsesHandler::parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, +absl::Status OpenAIResponsesHandler::parseRequestImpl(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) { absl::Status status = parseCommonPart(maxTokensLimit, bestOfLimit, maxModelLength); if (status != absl::OkStatus()) @@ -1045,72 +1045,34 @@ std::string OpenAIResponsesHandler::serializeUnaryResponseImpl(const std::vector // --- Unary response serialization --- -std::string OpenAIResponsesHandler::serializeUnaryResponse(const std::vector& generationOutputs) { +std::string OpenAIResponsesHandler::serializeUnaryResponse( + const std::vector& deltas, + ov::genai::GenerationFinishReason finishReason) { OVMS_PROFILE_FUNCTION(); - std::vector parsedOutputs; - usage.completionTokens = 0; - constexpr bool echo = false; // echo is not supported in Responses API - ov::genai::GenerationFinishReason responsesFinishReason = ov::genai::GenerationFinishReason::STOP; - for (const ov::genai::GenerationOutput& generationOutput : generationOutputs) { - updateUsage(usage, generationOutput.generated_ids, echo); - parsedOutputs.push_back(parseOutputIfNeeded(generationOutput.generated_ids)); - if (generationOutput.finish_reason == ov::genai::GenerationFinishReason::LENGTH) { - responsesFinishReason = ov::genai::GenerationFinishReason::LENGTH; - } - } - return serializeUnaryResponseImpl(parsedOutputs, responsesFinishReason); + ParsedOutput parsedOutput = parsedOutputFromDeltas(deltas); + return serializeUnaryResponseImpl({std::move(parsedOutput)}, finishReason); } -std::string OpenAIResponsesHandler::serializeUnaryResponse(ov::genai::EncodedResults& results) { +std::string OpenAIResponsesHandler::serializeUnaryResponse( + const std::vector>& allDeltas, + const std::vector& finishReasons, + const std::vector& /*logprobData*/) { OVMS_PROFILE_FUNCTION(); - usage.promptTokens = results.perf_metrics.get_num_input_tokens(); - usage.completionTokens = results.perf_metrics.get_num_generated_tokens(); - if (results.finish_reasons.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary LM responses generation result, defaulting to STOP"); - } - std::vector parsedOutputs; - ov::genai::GenerationFinishReason responsesFinishReason = ov::genai::GenerationFinishReason::STOP; - for (const auto& tokens : results.tokens) { - parsedOutputs.push_back(parseOutputIfNeeded(tokens)); - } - for (const auto& finishReason : results.finish_reasons) { - if (finishReason == ov::genai::GenerationFinishReason::LENGTH) { - responsesFinishReason = ov::genai::GenerationFinishReason::LENGTH; - break; - } - } - return serializeUnaryResponseImpl(parsedOutputs, responsesFinishReason); -} - -std::string OpenAIResponsesHandler::serializeUnaryResponse(ov::genai::VLMDecodedResults& results, const std::string& textResponse) { - OVMS_PROFILE_FUNCTION(); - usage.promptTokens = results.perf_metrics.get_num_input_tokens(); - usage.completionTokens = results.perf_metrics.get_num_generated_tokens(); - if (results.finish_reasons.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary VLM responses generation result, defaulting to STOP"); - } - // Usage is already correctly set from perf_metrics above — no need for updateUsage. + // Responses API does not expose logprobs; logprobData is intentionally unused. std::vector parsedOutputs; - if (!textResponse.empty()) { - if (outputParser != nullptr) { - // Same workaround as in chat completions - auto generatedTokens = encodeTextToTokens(textResponse); - parsedOutputs.push_back(parseOutputIfNeeded(generatedTokens)); - } else { - // Fast path: no output parser, use decoded text directly. - ParsedOutput output; - output.content = textResponse; - parsedOutputs.push_back(std::move(output)); - } - } - ov::genai::GenerationFinishReason responsesFinishReason = ov::genai::GenerationFinishReason::STOP; - for (const auto& finishReason : results.finish_reasons) { - if (finishReason == ov::genai::GenerationFinishReason::LENGTH) { - responsesFinishReason = ov::genai::GenerationFinishReason::LENGTH; - break; + parsedOutputs.reserve(allDeltas.size()); + for (const auto& deltas : allDeltas) { + parsedOutputs.push_back(parsedOutputFromDeltas(deltas)); + } + const ov::genai::GenerationFinishReason finishReason = [&]() { + for (const auto& fr : finishReasons) { + if (fr == ov::genai::GenerationFinishReason::LENGTH) { + return fr; + } } - } - return serializeUnaryResponseImpl(parsedOutputs, responsesFinishReason); + return finishReasons.empty() ? ov::genai::GenerationFinishReason::STOP : finishReasons[0]; + }(); + return serializeUnaryResponseImpl(parsedOutputs, finishReason); } // --- Streaming event building blocks --- @@ -1438,6 +1400,65 @@ std::string OpenAIResponsesHandler::serializeAudioDeltaEvent(const std::string& return buffer.GetString(); } +// --- Per-delta-type event composition for serializeStreamingChunk --- + +void OpenAIResponsesHandler::appendAudioDeltaEvents(const AudioDelta& d, std::vector& events) { + events.emplace_back(serializeAudioDeltaEvent(d.base64)); +} + +void OpenAIResponsesHandler::appendReasoningDeltaEvents(const ReasoningDelta& d, std::vector& events, const std::string& reasoningItemId) { + if (!responsesState.reasoningInitialized) { + events.emplace_back(serializeReasoningOutputItemAddedEvent(reasoningItemId)); + events.emplace_back(serializeReasoningSummaryPartAddedEvent(reasoningItemId)); + responsesState.reasoningInitialized = true; + } + responsesState.reasoningText += d.text; + events.emplace_back(serializeReasoningSummaryTextDeltaEvent(reasoningItemId, d.text)); +} + +void OpenAIResponsesHandler::appendContentDeltaEvents(const ContentDelta& d, std::vector& events, const std::string& outputItemId, const std::string& reasoningItemId) { + if (d.text.empty()) { + return; + } + if (responsesState.reasoningInitialized && !responsesState.reasoningCompleted) { + events.emplace_back(serializeReasoningSummaryTextDoneEvent(reasoningItemId)); + events.emplace_back(serializeReasoningSummaryPartDoneEvent(reasoningItemId)); + events.emplace_back(serializeReasoningOutputItemDoneEvent(reasoningItemId)); + responsesState.reasoningCompleted = true; + } + const uint64_t msgIdx = responsesState.reasoningInitialized ? 1 : 0; + if (!responsesState.messageInitialized) { + events.emplace_back(serializeOutputItemAddedEvent(outputItemId, msgIdx)); + events.emplace_back(serializeContentPartAddedEvent(outputItemId, msgIdx)); + responsesState.messageInitialized = true; + } + responsesState.outputText += d.text; + events.emplace_back(serializeOutputTextDeltaEvent(outputItemId, d.text, msgIdx)); +} + +void OpenAIResponsesHandler::appendToolCallDeltaEvents(const ToolCallDelta& d, std::vector& events, const std::string& reasoningItemId) { + if (responsesState.reasoningInitialized && !responsesState.reasoningCompleted) { + events.emplace_back(serializeReasoningSummaryTextDoneEvent(reasoningItemId)); + events.emplace_back(serializeReasoningSummaryPartDoneEvent(reasoningItemId)); + events.emplace_back(serializeReasoningOutputItemDoneEvent(reasoningItemId)); + responsesState.reasoningCompleted = true; + } + const uint64_t baseIdx = responsesState.reasoningInitialized ? 1 : 0; + const uint64_t tcOutputIdx = baseIdx + static_cast(d.index); + if (d.name) { + while (static_cast(responsesState.toolCalls.size()) <= d.index) + responsesState.toolCalls.push_back(ToolCall{}); + responsesState.toolCalls[d.index].id = d.id ? *d.id : ""; + responsesState.toolCalls[d.index].name = *d.name; + responsesState.toolCalls[d.index].arguments = ""; + events.emplace_back(serializeFunctionCallOutputItemAddedEvent(responsesState.toolCalls[d.index], tcOutputIdx)); + } + if (!d.arguments.empty() && static_cast(responsesState.toolCalls.size()) > d.index) { + responsesState.toolCalls[d.index].arguments += d.arguments; + events.emplace_back(serializeFunctionCallArgumentsDeltaEvent(responsesState.toolCalls[d.index].id, d.arguments, tcOutputIdx)); + } +} + // --- Top-level streaming methods --- std::string OpenAIResponsesHandler::serializeStreamingCreatedEvent() { @@ -1472,7 +1493,7 @@ std::string OpenAIResponsesHandler::serializeStreamingInProgressEvent() { return buffer.GetString(); } -std::string OpenAIResponsesHandler::serializeStreamingChunk(rapidjson::Document parsedDelta, ov::genai::GenerationFinishReason finishReason) { +std::string OpenAIResponsesHandler::serializeStreamingChunk(Delta delta, ov::genai::GenerationFinishReason finishReason) { OVMS_PROFILE_FUNCTION(); const auto createdAt = std::chrono::duration_cast(created.time_since_epoch()).count(); const std::string responseId = "resp-" + std::to_string(createdAt); @@ -1490,96 +1511,14 @@ std::string OpenAIResponsesHandler::serializeStreamingChunk(rapidjson::Document events.emplace_back(std::move(inProgressEvent)); } - // parsedDelta is a pre-parsed Document produced by OVMSTextStreamer::flushChunk or AudioStreamer. - // Shape: {"delta":{...}} for content/reasoning/tool_calls, or an empty Document{} - // for finish-only chunks, or {"_audio_delta":""} for audio chunks. - if (parsedDelta.HasMember("_audio_delta") && parsedDelta["_audio_delta"].IsString()) { - // Audio streaming chunk from speech_streamer - const std::string audioB64 = parsedDelta["_audio_delta"].GetString(); - events.emplace_back(serializeAudioDeltaEvent(audioB64)); - } else if (parsedDelta.HasMember("delta") && parsedDelta["delta"].IsObject()) { - const auto& deltaObj = parsedDelta["delta"]; - if (deltaObj.HasMember("reasoning_content") && deltaObj["reasoning_content"].IsString()) { - // Reasoning chunk - if (!responsesState.reasoningInitialized) { - events.emplace_back(serializeReasoningOutputItemAddedEvent(reasoningItemId)); - events.emplace_back(serializeReasoningSummaryPartAddedEvent(reasoningItemId)); - responsesState.reasoningInitialized = true; - } - const std::string reasoningText = deltaObj["reasoning_content"].GetString(); - responsesState.reasoningText += reasoningText; - events.emplace_back(serializeReasoningSummaryTextDeltaEvent(reasoningItemId, reasoningText)); - } else if (deltaObj.HasMember("content") && deltaObj["content"].IsString()) { - const std::string contentText = deltaObj["content"].GetString(); - if (!contentText.empty()) { - // Content chunk - close reasoning if it was active, init message if needed - if (responsesState.reasoningInitialized && !responsesState.reasoningCompleted) { - events.emplace_back(serializeReasoningSummaryTextDoneEvent(reasoningItemId)); - events.emplace_back(serializeReasoningSummaryPartDoneEvent(reasoningItemId)); - events.emplace_back(serializeReasoningOutputItemDoneEvent(reasoningItemId)); - responsesState.reasoningCompleted = true; - } - const uint64_t msgIdx = responsesState.reasoningInitialized ? 1 : 0; - if (!responsesState.messageInitialized) { - events.emplace_back(serializeOutputItemAddedEvent(outputItemId, msgIdx)); - events.emplace_back(serializeContentPartAddedEvent(outputItemId, msgIdx)); - responsesState.messageInitialized = true; - } - responsesState.outputText += contentText; - events.emplace_back(serializeOutputTextDeltaEvent(outputItemId, contentText, msgIdx)); - } - } else if (deltaObj.HasMember("tool_calls") && deltaObj["tool_calls"].IsArray()) { - // Tool call chunk - close reasoning if active - if (responsesState.reasoningInitialized && !responsesState.reasoningCompleted) { - events.emplace_back(serializeReasoningSummaryTextDoneEvent(reasoningItemId)); - events.emplace_back(serializeReasoningSummaryPartDoneEvent(reasoningItemId)); - events.emplace_back(serializeReasoningOutputItemDoneEvent(reasoningItemId)); - responsesState.reasoningCompleted = true; - } - const auto& toolCallsArr = deltaObj["tool_calls"]; - for (rapidjson::SizeType i = 0; i < toolCallsArr.Size(); ++i) { - const auto& tc = toolCallsArr[i]; - int tcIndex = tc.HasMember("index") ? tc["index"].GetInt() : 0; - // Determine the output index for this tool call - const uint64_t baseIdx = responsesState.reasoningInitialized ? 1 : 0; - const uint64_t tcOutputIdx = baseIdx + static_cast(tcIndex); - // Determine if this is a new tool call (has function name) - bool isNewToolCall = false; - std::string funcName; - std::string tcId; - std::string argDelta; - if (tc.HasMember("function") && tc["function"].IsObject()) { - const auto& funcObj = tc["function"]; - if (funcObj.HasMember("name") && funcObj["name"].IsString()) { - funcName = funcObj["name"].GetString(); - isNewToolCall = true; - } - if (funcObj.HasMember("arguments") && funcObj["arguments"].IsString()) { - argDelta = funcObj["arguments"].GetString(); - } - } - if (tc.HasMember("id") && tc["id"].IsString()) { - tcId = tc["id"].GetString(); - } - if (isNewToolCall) { - // Ensure we have enough entries in our tracking vector - while (static_cast(responsesState.toolCalls.size()) <= tcIndex) { - responsesState.toolCalls.push_back(ToolCall{}); - } - responsesState.toolCalls[tcIndex].id = tcId; - responsesState.toolCalls[tcIndex].name = funcName; - responsesState.toolCalls[tcIndex].arguments = ""; - events.emplace_back(serializeFunctionCallOutputItemAddedEvent(responsesState.toolCalls[tcIndex], tcOutputIdx)); - } - if (!argDelta.empty() && static_cast(responsesState.toolCalls.size()) > tcIndex) { - responsesState.toolCalls[tcIndex].arguments += argDelta; - events.emplace_back(serializeFunctionCallArgumentsDeltaEvent(responsesState.toolCalls[tcIndex].id, argDelta, tcOutputIdx)); - } - } - } - // Empty delta object (no recognized member) — finish-only chunk, no events to emit here. - } - // Empty Document (no "delta" member) — finish-only chunk; lifecycle events already emitted above. + std::visit(overloaded{ + [&](const AudioDelta& d) { appendAudioDeltaEvents(d, events); }, + [&](const ReasoningDelta& d) { appendReasoningDeltaEvents(d, events, reasoningItemId); }, + [&](const ContentDelta& d) { appendContentDeltaEvents(d, events, outputItemId, reasoningItemId); }, + [&](const ToolCallDelta& d) { appendToolCallDeltaEvents(d, events, reasoningItemId); }, + [&](const FinishDelta&) {}, + }, + delta); if (finishReason != ov::genai::GenerationFinishReason::NONE) { // Close any open reasoning that wasn't closed by content transition diff --git a/src/llm/apis/openai_responses.hpp b/src/llm/apis/openai_responses.hpp index 5b10908521..41962d6346 100644 --- a/src/llm/apis/openai_responses.hpp +++ b/src/llm/apis/openai_responses.hpp @@ -92,16 +92,23 @@ class OpenAIResponsesHandler : public OpenAIApiHandler { // Audio streaming event serializers std::string serializeAudioDeltaEvent(const std::string& audioB64); + // Per-delta-type event composition for serializeStreamingChunk + void appendAudioDeltaEvents(const AudioDelta& delta, std::vector& events); + void appendReasoningDeltaEvents(const ReasoningDelta& delta, std::vector& events, const std::string& reasoningItemId); + void appendContentDeltaEvents(const ContentDelta& delta, std::vector& events, const std::string& outputItemId, const std::string& reasoningItemId); + void appendToolCallDeltaEvents(const ToolCallDelta& delta, std::vector& events, const std::string& reasoningItemId); + public: using OpenAIApiHandler::OpenAIApiHandler; // Inherit constructors - absl::Status parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, - std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt) override; + absl::Status parseRequestImpl(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, + std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) override; - std::string serializeUnaryResponse(const std::vector& generationOutputs) override; - std::string serializeUnaryResponse(ov::genai::EncodedResults& results) override; - std::string serializeUnaryResponse(ov::genai::VLMDecodedResults& results, const std::string& textResponse) override; - std::string serializeStreamingChunk(rapidjson::Document parsedDelta, ov::genai::GenerationFinishReason finishReason) override; + std::string serializeUnaryResponse(const std::vector& deltas, ov::genai::GenerationFinishReason finishReason) override; + std::string serializeUnaryResponse(const std::vector>& allDeltas, + const std::vector& finishReasons, + const std::vector& logprobData) override; + std::string serializeStreamingChunk(Delta delta, ov::genai::GenerationFinishReason finishReason) override; std::string serializeStreamingUsageChunk() override; std::string serializeStreamingHandshakeChunk() override; std::string serializeStreamingCreatedEvent() override; diff --git a/src/llm/io_processing/base_output_parser.cpp b/src/llm/io_processing/base_output_parser.cpp index 2091488779..e4969ce560 100644 --- a/src/llm/io_processing/base_output_parser.cpp +++ b/src/llm/io_processing/base_output_parser.cpp @@ -19,10 +19,6 @@ #include #include -#include "src/port/rapidjson_document.hpp" -#include "src/port/rapidjson_stringbuffer.hpp" -#include "src/port/rapidjson_writer.hpp" - #include "base_output_parser.hpp" #include "utils.hpp" @@ -72,45 +68,27 @@ ToolsParameterTypeMap_t createToolsParametersTypesMap(const ToolsSchemas_t& tool return toolsParametersTypes; } -rapidjson::Document BaseOutputParser::wrapFirstDelta(const std::string& functionName, int toolCallIndex) { - rapidjson::Document wrappedDelta; - wrappedDelta.SetObject(); - rapidjson::Value toolCalls(rapidjson::kArrayType); - rapidjson::Value toolCallObj(rapidjson::kObjectType); - rapidjson::Value idValue(generateRandomId().c_str(), wrappedDelta.GetAllocator()); - toolCallObj.AddMember("id", idValue, wrappedDelta.GetAllocator()); - toolCallObj.AddMember("type", "function", wrappedDelta.GetAllocator()); - toolCallObj.AddMember("index", toolCallIndex, wrappedDelta.GetAllocator()); - rapidjson::Value functionObj(rapidjson::kObjectType); - rapidjson::Value nameValue(functionName.c_str(), wrappedDelta.GetAllocator()); - functionObj.AddMember("name", nameValue, wrappedDelta.GetAllocator()); - - toolCallObj.AddMember("function", functionObj, wrappedDelta.GetAllocator()); - toolCalls.PushBack(toolCallObj, wrappedDelta.GetAllocator()); - rapidjson::Value deltaWrapper(rapidjson::kObjectType); - deltaWrapper.AddMember("tool_calls", toolCalls, wrappedDelta.GetAllocator()); - wrappedDelta.AddMember("delta", deltaWrapper, wrappedDelta.GetAllocator()); - return wrappedDelta; -} +std::string BaseOutputParser::buildParsingConfigStringRepresentation() const { + std::string result = "StartTags: ["; + for (const auto& tag : parsingConfig.startTags) { + result += tag + ", "; + } + result += "], EndTag: " + parsingConfig.endTag + ", ContentTagsToErase: ["; + for (const auto& tag : parsingConfig.stringsToErase) { + result += tag + ", "; + } + result += "]"; -rapidjson::Document BaseOutputParser::wrapDelta(const rapidjson::Document& delta, int toolCallIndex) { - rapidjson::Document wrappedDelta; - wrappedDelta.SetObject(); - rapidjson::Value toolCalls(rapidjson::kArrayType); - rapidjson::Value toolCallObj(rapidjson::kObjectType); - toolCallObj.AddMember("index", toolCallIndex, wrappedDelta.GetAllocator()); - rapidjson::Value functionObj(rapidjson::kObjectType); - for (auto it = delta.MemberBegin(); it != delta.MemberEnd(); ++it) { - rapidjson::Value key(it->name, wrappedDelta.GetAllocator()); - rapidjson::Value value(it->value, wrappedDelta.GetAllocator()); - functionObj.AddMember(key, value, wrappedDelta.GetAllocator()); + // Additionally include the resolved start token IDs and their corresponding tags in the string representation + result += ", ResolvedStartTokenToTag: {"; + for (const auto& [tokenId, tag] : resolvedStartTokenToTag) { + result += std::to_string(tokenId) + ": " + tag + ", "; } - toolCallObj.AddMember("function", functionObj, wrappedDelta.GetAllocator()); - toolCalls.PushBack(toolCallObj, wrappedDelta.GetAllocator()); - rapidjson::Value deltaWrapper(rapidjson::kObjectType); - deltaWrapper.AddMember("tool_calls", toolCalls, wrappedDelta.GetAllocator()); - wrappedDelta.AddMember("delta", deltaWrapper, wrappedDelta.GetAllocator()); - return wrappedDelta; + result += "}"; + + result += ", ImplicitStart: " + std::string(implicitStart ? "true" : "false"); + result += ", NeedsSpecialTokens: " + std::string(parsingConfig.needsSpecialTokens ? "true" : "false"); + return result; } } // namespace ovms diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index 0b83ea3839..163a836260 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -25,9 +25,9 @@ #include #include +#include "delta.hpp" +#include "output_parsing_config.hpp" #include "src/port/rapidjson_document.hpp" -#include "src/port/rapidjson_stringbuffer.hpp" -#include "src/port/rapidjson_writer.hpp" #include "src/llm/apis/tool_schema_wrapper.hpp" @@ -69,70 +69,57 @@ ToolsParameterTypeMap_t createToolsParametersTypesMap(const ToolsSchemas_t& tool class BaseOutputParser { protected: ov::genai::Tokenizer tokenizer; + + // Parsing configuration set by sub-class constructors. + OutputParsingConfig parsingConfig; + + // Token IDs resolved from parsingConfig.tokenIdStartTags on construction. + // Maps token_id -> tag_string so the OutputParser can synthesise the boundary + // text when a token-ID-based phase transition fires. + std::unordered_map resolvedStartTokenToTag; + // When true, the chat template has already emitted the parser's start tag as the // trailing tokens of the prompt, so the model output is expected to begin already // inside the parsed segment (e.g. reasoning) without producing the start tag itself. - // Used by reasoning parsers for models like Qwen3.6, Qwen3-VL. - // append "\n" at the end of the prompt when thinking is enabled. bool implicitStart = false; + // Called once from the constructor. + void resolveSpecialTokenIds() { + for (const auto& tag : parsingConfig.tokenIdStartTags) { + if (tag.empty()) + continue; + const auto tensor = tokenizer.encode(tag, ov::genai::add_special_tokens(false)).input_ids; + if (tensor.get_size() == 1) { + resolvedStartTokenToTag[tensor.data()[0]] = tag; + } + } + } + public: BaseOutputParser() = delete; explicit BaseOutputParser(ov::genai::Tokenizer& tokenizer) : tokenizer(tokenizer) {} + + explicit BaseOutputParser(ov::genai::Tokenizer& tokenizer, OutputParsingConfig config) : + tokenizer(tokenizer), + parsingConfig(std::move(config)) { + resolveSpecialTokenIds(); + } + virtual ~BaseOutputParser() = default; + virtual void resetState() {} + void setImplicitStart(bool value) { implicitStart = value; } bool isImplicitStart() const { return implicitStart; } - // Common function to wrap first delta with full function name in a JSON object that conforms to OpenAI API response format: - // {"tool_calls":[{"id": , "type": "function", "index":,"function":}]} - static rapidjson::Document wrapFirstDelta(const std::string& functionName, int toolCallIndex); - // Common function to wrap subsequent deltas in a JSON object that conforms to OpenAI API response format - // {"tool_calls":[{"index":0,"function":}]} - static rapidjson::Document wrapDelta(const rapidjson::Document& delta, int toolCallIndex); + const OutputParsingConfig& getParsingConfig() const { return parsingConfig; } + const std::unordered_map& getResolvedStartTokenToTag() const { return resolvedStartTokenToTag; } - // --- Specialized output parsers interface --- + std::string buildParsingConfigStringRepresentation() const; - // Parse model output and extract relevant information to parsedOutput fields. Raw generated tokens are provided as an argument. - // Additionally parsedOutput.content is already filled with decoded content when this method is called, enabling chain or parsing. - // Parser is also responsible for removing extracted part from the parsedOutput.content if necessary. - virtual void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) = 0; - - // Parse model output chunk in the streaming mode. If in result of processing the chunk we cannot produce meaningful response, we return std::nullopt. - // Otherwise we return a JSON object containing the delta that conforms to OpenAI API. - // tokens holds the token IDs that produced chunkResponse (may be empty; currently informational for future use). - virtual std::optional parseChunk(const std::string& chunkResponse, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) = 0; - - // Get the tags that marks the beginning of the segment that should be processed by the parser. - // This method is used in streaming mode to determine if the parser should start processing the content. - // If empty string is returned, it means that the parser will never start processing the content. - virtual const std::vector& getParsingStartTags() const = 0; - - // Get a vector of additional tags that mark beginning of the segment that should be processed by the parser. - // These tags are considered only if they are the first output produced by the model. - // In streaming mode it means that they are considered only in UNKNOWN phase. - virtual const std::vector& getSpecialParsingStartTags() const = 0; - - // Get the tag that marks the end of the segment that should be processed by the parser. - // This method is used in streaming mode to determine if the parser should stop processing the content. - // If empty string is returned, it means that the parser will keep processing until the end of the content. - virtual const std::string& getParsingEndTag() const = 0; - - // Indicates whether the parser requires special tokens to be present in the streaming output. - // If true, the tokenizer used in the TextStreamer should be configured to not skip special tokens. - // This is important for parsers that rely on special tokens to identify parsing boundaries or - // specific segments of the output. - virtual bool requiresStreamingWithSpecialTokens() const { - return false; - } + // --- Specialized output parsers interface --- - // Get the vector of special tags that should be erased from the content before parsing. - // This is useful for cleaning up the content from tags that are necessary for parsing - // but should not be present in the final output. - virtual const std::vector& getSpecialTagsToErase() const { - static const std::vector emptyVector; - return emptyVector; - } + virtual std::optional parseChunk(const std::string& chunkResponse, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) = 0; }; } // namespace ovms diff --git a/src/llm/io_processing/default_content_parser.cpp b/src/llm/io_processing/default_content_parser.cpp new file mode 100644 index 0000000000..0e48ded49f --- /dev/null +++ b/src/llm/io_processing/default_content_parser.cpp @@ -0,0 +1,62 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "src/stringutils.hpp" + +#include "default_content_parser.hpp" + +namespace ovms { + +DefaultContentParser::DefaultContentParser(ov::genai::Tokenizer& tokenizer, + std::vector stringsToErase) : + BaseOutputParser(tokenizer, [&] { + OutputParsingConfig cfg; + cfg.stringsToErase = std::move(stringsToErase); + return cfg; + }()) {} + +std::optional DefaultContentParser::parseChunk( + const std::string& buf, + const std::vector& /*tokens*/, + ov::genai::GenerationFinishReason /*finishReason*/) { + + bool anyComplete = false; + for (const auto& tag : parsingConfig.stringsToErase) { + if (buf.find(tag) != std::string::npos) + anyComplete = true; + } + if (!anyComplete) { + for (const auto& tag : parsingConfig.stringsToErase) { + if (stringsOverlap(buf, tag)) + return std::nullopt; // partial match — hold + } + } + + std::string content = buf; + if (anyComplete) { + for (const auto& tag : parsingConfig.stringsToErase) { + size_t pos = 0; + while ((pos = content.find(tag, pos)) != std::string::npos) + content.erase(pos, tag.size()); + } + } + + return ContentDelta{std::move(content)}; +} + +} // namespace ovms diff --git a/src/llm/io_processing/default_content_parser.hpp b/src/llm/io_processing/default_content_parser.hpp new file mode 100644 index 0000000000..b6fe83f4c9 --- /dev/null +++ b/src/llm/io_processing/default_content_parser.hpp @@ -0,0 +1,42 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "base_output_parser.hpp" + +namespace ovms { + +// Generic content parser: strips model-specific structural tokens or any other strings that should not be included in the final content output. +// (e.g. BOS/EOS for minicpm5, chat-template turn markers for gemma4/lfm2). +// Parsers that need richer hold logic (e.g. Onyx) provide their own content parser subclass. +class DefaultContentParser final : public BaseOutputParser { +public: + DefaultContentParser() = delete; + explicit DefaultContentParser(ov::genai::Tokenizer& tokenizer, + std::vector stringsToErase = {}); + + std::optional parseChunk(const std::string& buffer, + const std::vector& tokens, + ov::genai::GenerationFinishReason finishReason) override; +}; + +} // namespace ovms diff --git a/src/llm/io_processing/delta.hpp b/src/llm/io_processing/delta.hpp new file mode 100644 index 0000000000..7799d193a9 --- /dev/null +++ b/src/llm/io_processing/delta.hpp @@ -0,0 +1,57 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +namespace ovms { + +struct ContentDelta { + std::string text; +}; + +struct ReasoningDelta { + std::string text; +}; + +struct ToolCallDelta { + int index; + // Present only on the first delta for a given tool call index; nullopt on argument-streaming deltas. + std::optional id; + std::optional name; + std::string arguments; +}; + +// Emitted when generation ends on a swallowed token: no content to carry, but the +// caller still needs to emit a finish_reason chunk. +struct FinishDelta {}; + +// Audio streaming chunk (omni-model path only): base64-encoded PCM16 audio. +struct AudioDelta { + std::string base64; +}; + +using Delta = std::variant; + +// Helper for exhaustive std::visit — CTAD deduction guide included. +template +struct overloaded : Ts... { + using Ts::operator()...; +}; +template +overloaded(Ts...)->overloaded; + +} // namespace ovms diff --git a/src/llm/io_processing/devstral/tool_parser.cpp b/src/llm/io_processing/devstral/tool_parser.cpp index 6ceb74a42d..d0deb7ae66 100644 --- a/src/llm/io_processing/devstral/tool_parser.cpp +++ b/src/llm/io_processing/devstral/tool_parser.cpp @@ -19,7 +19,6 @@ #include #include -#include "src/port/rapidjson_document.hpp" #include "src/logging.hpp" #include "src/llm/io_processing/utils.hpp" #include "src/stringutils.hpp" @@ -27,114 +26,21 @@ namespace ovms { -void DevstralToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // expected format: [TOOL_CALLS]tool_name[ARGS]{"arg1": "value1", ...} - if (parsedOutput.content.empty() || generatedTokens.size() <= 0) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "No content to parse for tool calls"); - return; - } - size_t firstToolTokenIndex; - auto it = std::find(generatedTokens.begin(), generatedTokens.end(), this->botTokenId); - if (it != generatedTokens.end()) { - firstToolTokenIndex = std::distance(generatedTokens.begin(), it); - } else { - return; - } - - size_t firstArgsTokenIndex; - auto itArgs = std::find(generatedTokens.begin() + firstToolTokenIndex, generatedTokens.end(), this->argsTokenId); - if (itArgs != generatedTokens.end()) { - firstArgsTokenIndex = std::distance(generatedTokens.begin(), itArgs); - } else { - return; - } - if (firstToolTokenIndex > firstArgsTokenIndex) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "First tool token index is greater than first args token index."); - return; - } - std::vector toolNameTokens(generatedTokens.begin() + (firstToolTokenIndex + 1), generatedTokens.begin() + (firstArgsTokenIndex)); - std::vector argumentsTokens(generatedTokens.begin() + (firstArgsTokenIndex + 1), generatedTokens.end()); - - ToolCall toolCall; - std::string toolName = tokenizer.decode(toolNameTokens, ov::AnyMap{ov::genai::skip_special_tokens(true)}); - std::string arguments = tokenizer.decode(argumentsTokens, ov::AnyMap{ov::genai::skip_special_tokens(true)}); - ovms::trim(toolName); // trim in case of extra spaces/newlines - toolCall.name = toolName; - if (arguments.empty()) { - arguments = "{}"; // set empty arguments to {} - } - toolCall.arguments = arguments; - toolCall.id = generateRandomId(); // Generate a random ID for the tool call - parsedOutput.toolCalls.push_back(toolCall); - - // get subset of generatedTokens starting from begin() to firstArgsTokenIndex - std::vector contentTokens; - if (firstToolTokenIndex > 0) { - contentTokens = std::vector(generatedTokens.begin(), generatedTokens.begin() + firstToolTokenIndex); - parsedOutput.content = tokenizer.decode(contentTokens, ov::AnyMap{ov::genai::skip_special_tokens(true)}); // Return only the content till tool call - } else { - parsedOutput.content = tokenizer.decode(contentTokens, ov::AnyMap{ov::genai::skip_special_tokens(true)}); - } - return; +std::optional DevstralToolParser::sendFullDelta(ToolCall& toolCall) { + return ToolCallDelta{this->toolCallIndex, std::nullopt, std::nullopt, toolCall.arguments}; } -std::optional DevstralToolParser::sendFullDelta(ToolCall& toolCall) { - rapidjson::Document argsDelta; - argsDelta.Parse(toolCall.arguments.c_str()); - rapidjson::Document argumentsWrapper; - argumentsWrapper.SetObject(); - rapidjson::Document::AllocatorType& allocator = argumentsWrapper.GetAllocator(); - // now we need to add string toolCall.arguments to argumentsWrapper under "arguments" key - rapidjson::Value toolCallsString(rapidjson::kStringType); - toolCallsString.SetString(toolCall.arguments.c_str(), allocator); - argumentsWrapper.AddMember("arguments", toolCallsString, allocator); - auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); - return currentDelta; +ToolCallDelta DevstralToolParser::wrapCombinedDelta(ToolCall& toolCall) { + return ToolCallDelta{this->toolCallIndex, generateRandomId(), toolCall.name, toolCall.arguments}; } -rapidjson::Document DevstralToolParser::wrapCombinedDelta(ToolCall& toolCall) { - rapidjson::Document wrappedDelta; - wrappedDelta.SetObject(); - rapidjson::Value toolCalls(rapidjson::kArrayType); - rapidjson::Value toolCallObj(rapidjson::kObjectType); - rapidjson::Value idValue(generateRandomId().c_str(), wrappedDelta.GetAllocator()); - rapidjson::Value toolCallsString(rapidjson::kStringType); - - toolCallObj.AddMember("id", idValue, wrappedDelta.GetAllocator()); - toolCallObj.AddMember("type", "function", wrappedDelta.GetAllocator()); - toolCallObj.AddMember("index", toolCallIndex, wrappedDelta.GetAllocator()); - rapidjson::Value functionObj(rapidjson::kObjectType); - rapidjson::Value nameValue(toolCall.name.c_str(), wrappedDelta.GetAllocator()); - functionObj.AddMember("name", nameValue, wrappedDelta.GetAllocator()); - // now we need to add string toolCall.arguments to argumentsWrapper under "arguments" key - - toolCallsString.SetString(toolCall.arguments.c_str(), wrappedDelta.GetAllocator()); - functionObj.AddMember("arguments", toolCallsString, wrappedDelta.GetAllocator()); - toolCallObj.AddMember("function", functionObj, wrappedDelta.GetAllocator()); - toolCalls.PushBack(toolCallObj, wrappedDelta.GetAllocator()); - rapidjson::Value deltaWrapper(rapidjson::kObjectType); - deltaWrapper.AddMember("tool_calls", toolCalls, wrappedDelta.GetAllocator()); - wrappedDelta.AddMember("delta", deltaWrapper, wrappedDelta.GetAllocator()); - return wrappedDelta; -} - -rapidjson::Document DevstralToolParser::parseContentChunk() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("content"); - writer.String(streamContent.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); +ContentDelta DevstralToolParser::parseContentChunk() { + ContentDelta d{std::move(streamContent)}; streamContent.clear(); - return doc; + return d; } -std::optional DevstralToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional DevstralToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { /* Devstral [TOOL_CALL]tool_name[ARGS]arguments[] It does not support parallel tool calls, so tool calls are always in sequence. @@ -147,6 +53,13 @@ std::optional DevstralToolParser::parseChunk(const std::str We store the history of chunks in streamContent string. After state changes are detected, we clear the streamContent to only keep unprocessed part. */ + // Ignore no-op empty chunks when there is nothing buffered to flush. + // Keep processing empty STOP chunks only when streamContent already holds + // pending argument text (missing end-tag finalization path). + if (chunk.empty() && this->streamContent.empty()) { + return std::nullopt; + } + this->streamContent += chunk; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Chunk content: '{}', StreamContent: '{}', State: {}", chunk, this->streamContent, std::to_string(this->internalState)); if (this->internalState == AWAITING_START_TAG) { @@ -175,7 +88,20 @@ std::optional DevstralToolParser::parseChunk(const std::str } if (this->internalState == AWAITING_ARGS_TAG) { size_t pos = this->streamContent.find(this->parsingArgsStartTag); - if (pos != std::string::npos) { + if (pos == std::string::npos) { + // [ARGS] not found — check if generation has ended (end tag or finish reason). + // Flush whatever accumulated as plain content so it is not silently dropped. + size_t endPos = this->streamContent.find(this->parsingEndTag); + if (endPos != std::string::npos || finishReason != ov::genai::GenerationFinishReason::NONE) { + if (endPos != std::string::npos) { + this->streamContent = this->streamContent.substr(0, endPos); + } + if (!this->streamContent.empty()) { + return parseContentChunk(); + } + } + return std::nullopt; + } else { this->internalState = PROCESSING_ARGS; this->toolName = this->streamContent.substr(0, pos); ovms::trim(this->toolName); // trim in case of extra spaces/newlines @@ -195,11 +121,10 @@ std::optional DevstralToolParser::parseChunk(const std::str this->streamContent = ""; return wrapCombinedDelta(toolCall); } else { - return wrapFirstDelta(this->toolName, this->toolCallIndex); + return ToolCallDelta{this->toolCallIndex, generateRandomId(), this->toolName, ""}; } - } else { - return std::nullopt; } + return std::nullopt; } if (this->internalState == PROCESSING_ARGS) { size_t endPos = this->streamContent.find(this->parsingEndTag); @@ -210,6 +135,12 @@ std::optional DevstralToolParser::parseChunk(const std::str arguments = this->streamContent; } + // Suppress the spurious "{}" delta that would otherwise be appended to the accumulated arguments. + if (arguments.empty() && argumentsEmitted) { + this->streamContent = ""; + return std::nullopt; + } + ToolCall toolCall; if (!arguments.empty()) toolCall.arguments = arguments; @@ -217,6 +148,7 @@ std::optional DevstralToolParser::parseChunk(const std::str toolCall.arguments = "{}"; toolCall.name = this->toolName; this->streamContent = ""; + argumentsEmitted = !arguments.empty(); return sendFullDelta(toolCall); } return std::nullopt; diff --git a/src/llm/io_processing/devstral/tool_parser.hpp b/src/llm/io_processing/devstral/tool_parser.hpp index 5a591696ab..7bb0c78496 100644 --- a/src/llm/io_processing/devstral/tool_parser.hpp +++ b/src/llm/io_processing/devstral/tool_parser.hpp @@ -18,11 +18,10 @@ #include #include #include +#include #include -#include "src/port/rapidjson_document.hpp" #include "src/llm/io_processing/base_output_parser.hpp" -#include "src/llm/io_processing/partial_json_builder.hpp" #include "src/llm/apis/tool_schema_wrapper.hpp" namespace ovms { @@ -47,34 +46,41 @@ class DevstralToolParser : public BaseOutputParser { int toolCallIndex = -1; std::string streamContent = ""; // content accumulated from stream chunks std::string toolName = ""; - std::optional sendFullDelta(ToolCall& toolCall); + bool argumentsEmitted = false; + std::optional sendFullDelta(ToolCall& toolCall); public: DevstralToolParser() = delete; - DevstralToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : - BaseOutputParser(tokenizer), - toolSchemas(toolSchemas) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - rapidjson::Document parseContentChunk(); - rapidjson::Document wrapCombinedDelta(ToolCall& toolCall); - const std::vector& getParsingStartTags() const override { - static const std::vector toolCallStartTags{parsingToolCallsStartTag}; - return toolCallStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - // Tools calls are expected to be the last part of the content, so we do not specify an end tag. - const std::string& getParsingEndTag() const override { - return this->parsingEndTag; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + // [TOOL_CALLS] is always visible as text (needsSpecialTokens=true). + // Put it in startTags for reliable text-based detection. + cfg.startTags = {"[TOOL_CALLS]"}; + cfg.tokenIdStartTags = {"[TOOL_CALLS]"}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - bool requiresStreamingWithSpecialTokens() const override { - return true; + DevstralToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()), + toolSchemas(toolSchemas) {} + + void resetState() override { + internalState = AWAITING_START_TAG; + toolCallIndex = -1; + streamContent.clear(); + toolName.clear(); + argumentsEmitted = false; } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + ContentDelta parseContentChunk(); + ToolCallDelta wrapCombinedDelta(ToolCall& toolCall); }; } // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp index cd1077adb2..303423f5a5 100644 --- a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp @@ -18,11 +18,8 @@ #include #include -#include "src/port/rapidjson_document.hpp" - #include "../../../logging.hpp" #include "gemma4_reasoning_parser.hpp" -#include "../utils.hpp" namespace ovms { void Gemma4ReasoningParser::skipToken(const std::vector& generatedTokens, size_t& pos, int64_t tokenId) { @@ -31,51 +28,16 @@ void Gemma4ReasoningParser::skipToken(const std::vector& generatedToken } } -void Gemma4ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - auto startPos = std::string::npos; - auto endPos = std::string::npos; - - auto startIt = std::find(generatedTokens.begin(), generatedTokens.end(), channelStartTokenId); - auto endIt = std::find(generatedTokens.begin(), generatedTokens.end(), channelEndTokenId); - - if (startIt != generatedTokens.end() && endIt != generatedTokens.end() && startIt < endIt) { - startPos = std::distance(generatedTokens.begin(), startIt); - endPos = std::distance(generatedTokens.begin(), endIt); - } - - if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { - skipToken(generatedTokens, startPos, channelStartTokenId); - std::string reasoningText = tokenizer.decode(std::vector(generatedTokens.begin() + startPos, generatedTokens.begin() + endPos), ov::genai::skip_special_tokens(true)); - if (reasoningText.find(reasoningStrIndicator) == 0) { - reasoningText = reasoningText.substr(reasoningStrIndicator.size()); - } - parsedOutput.reasoning = reasoningText; - // Remove reasoning from content - std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); // content MUST never appear before reasoning - parsedOutput.content = contentWithoutReasoning; - } -} -std::optional Gemma4ReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional Gemma4ReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { if (chunk.empty()) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Gemma4ReasoningParser"); return std::nullopt; } - if (chunk.find(getParsingStartTags()[0]) != std::string::npos || chunk.find(getParsingEndTag()) != std::string::npos) { + if (chunk.find(parsingConfig.startTags[0]) != std::string::npos || chunk.find(parsingConfig.endTag) != std::string::npos) { return std::nullopt; } else { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("reasoning_content"); - writer.String(chunk.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - return doc; + return ReasoningDelta{chunk}; } return std::nullopt; } diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp index 887036a59d..54f8d69542 100644 --- a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp @@ -35,25 +35,20 @@ class Gemma4ReasoningParser : public Qwen3ReasoningParser { public: Gemma4ReasoningParser() = delete; - explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer) : - Qwen3ReasoningParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - - bool requiresStreamingWithSpecialTokens() const override { - return true; - } - - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{this->parsingStartTag}; - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - return parsingEndTag; + explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + Qwen3ReasoningParser(tokenizer, [&]() -> std::optional { + if (configOverride.has_value()) + return configOverride; + OutputParsingConfig cfg; + cfg.startTags = {"<|channel>thought\n"}; + cfg.tokenIdStartTags = {"<|channel>"}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + return cfg; + }()) { + resolveSpecialTokenIds(); } + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp b/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp index 058f0c1add..612a84a382 100644 --- a/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp +++ b/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp @@ -318,9 +318,8 @@ bool Gemma4ToolParser::parseInToolCallEndedState() { this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); this->currentState = State::AfterToolCall; } else { - this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); - this->currentState = State::AfterToolCall; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}, returning to content state", toolCallEndTagPos); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Waiting for more data in ToolCallEnded state; no complete next tool call prefix or end tag found from position: {}", this->streamingPosition); + return false; } return true; } @@ -345,32 +344,24 @@ bool Gemma4ToolParser::parseNewContent() { return false; } -std::optional Gemma4ToolParser::wrapDeltaContent(const std::string& content) { - if (content.empty() || content == "") { +std::optional Gemma4ToolParser::wrapDeltaContent(const std::string& content) { + if (content.empty()) return std::nullopt; - } - rapidjson::Document doc(rapidjson::kObjectType); - rapidjson::Value deltaObj(rapidjson::kObjectType); - deltaObj.AddMember("content", rapidjson::Value(content.c_str(), doc.GetAllocator()), doc.GetAllocator()); - doc.AddMember("delta", deltaObj, doc.GetAllocator()); - return doc; + return ContentDelta{content}; } -rapidjson::Document Gemma4ToolParser::wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { - rapidjson::Document doc(rapidjson::kObjectType); - doc.AddMember("arguments", rapidjson::Value(argsStr.c_str(), doc.GetAllocator()), doc.GetAllocator()); - - return BaseOutputParser::wrapDelta(doc, toolCallIndex); +ToolCallDelta Gemma4ToolParser::wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { + return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, argsStr}; } -std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { if (!chunk.empty()) { this->streamingContent += chunk; } if (parseNewContent()) { if (this->currentState == State::ToolCallParameters) { - return BaseOutputParser::wrapFirstDelta(this->toolCall.name, toolCallIndex); + return ToolCallDelta{toolCallIndex, generateRandomId(), this->toolCall.name, ""}; } if (this->currentState == State::ToolCallEnded) { return wrapDeltaArgs(this->toolCall.arguments, toolCallIndex); @@ -384,6 +375,17 @@ std::optional Gemma4ToolParser::parseChunk(const std::strin content = this->streamingContent.substr(this->streamingPosition); } this->streamingPosition += content.size(); + + if (finishReason != ov::genai::GenerationFinishReason::NONE) { + for (const std::string& tagToErase : {TURN_END_TAG, TOOL_RESPONSE_START_TAG}) { + size_t tagPos = content.find(tagToErase); + while (tagPos != std::string::npos) { + content.erase(tagPos, tagToErase.length()); + tagPos = content.find(tagToErase, tagPos); + } + } + } + return wrapDeltaContent(content); } if (this->currentState == State::AfterToolCall) { @@ -392,6 +394,14 @@ std::optional Gemma4ToolParser::parseChunk(const std::strin } if (finishReason != ov::genai::GenerationFinishReason::NONE) { + // Unary/STOP flush can arrive after a chunk that only advanced one state + // (e.g. parsed the tool name but not yet the immediately following "}"). + // Give the state machine one last chance to consume already-buffered data + // before deciding whether an arguments delta exists. + if (this->currentState == State::ToolCallParameters) { + parseToolCallParametersState(); + } + if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { return wrapDeltaArgs(this->toolCall.arguments, toolCallIndex); } @@ -400,7 +410,7 @@ std::optional Gemma4ToolParser::parseChunk(const std::strin auto content = this->streamingContent.substr(this->streamingPosition); this->streamingPosition += content.size(); - for (const std::string& tagToErase : getSpecialTagsToErase()) { + for (const std::string& tagToErase : {TURN_END_TAG, TOOL_RESPONSE_START_TAG}) { size_t tagPos = content.find(tagToErase); while (tagPos != std::string::npos) { content.erase(tagPos, tagToErase.length()); @@ -449,81 +459,4 @@ bool Gemma4ToolParser::parseSingleToolCall(const std::string& toolStr, ToolCall& return false; } -void Gemma4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - std::vector tools; - std::vector> toolCallPositions; - size_t pos = 0; - - while (pos != std::string::npos) { - size_t start = std::string::npos; - size_t end = std::string::npos; - - auto it = std::find(generatedTokens.begin() + pos, generatedTokens.end(), botTokenId); - if (it != generatedTokens.end()) { - start = std::distance(generatedTokens.begin(), it); - } else { - break; - } - auto itArgs = std::find(generatedTokens.begin() + start, generatedTokens.end(), eotTokenId); - if (itArgs != generatedTokens.end()) { - end = std::distance(generatedTokens.begin(), itArgs); - } else { - break; - } - - std::string toolCallStr = tokenizer.decode(std::vector(generatedTokens.begin() + start + 1, generatedTokens.begin() + end + 1), ov::AnyMap{ov::genai::skip_special_tokens(false)}); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool list string: {}", toolCallStr); - - while (!toolCallStr.empty()) { - size_t nextToolPos = toolCallStr.find(TOOL_CALL_NAME_PREFIX, TOOL_CALL_NAME_PREFIX.length()); - size_t toolEndPos; - if (nextToolPos == std::string::npos) { - toolEndPos = toolCallStr.rfind(TOOL_ARGS_END_INDICATOR); - } else { - toolEndPos = nextToolPos - 1; - } - std::string singleTool; - if (toolEndPos != std::string::npos) { - singleTool = toolCallStr.substr(0, toolEndPos + TOOL_ARGS_END_INDICATOR.length()); - if (toolEndPos + TOOL_ARGS_END_INDICATOR.length() < toolCallStr.length()) { - toolCallStr = toolCallStr.substr(toolEndPos + TOOL_ARGS_END_INDICATOR.length()); - } else { - toolCallStr.clear(); - } - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed single tool string {}", singleTool); - } else { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more tool strings found in the decoded string: {}", toolCallStr); - break; - } - - if (!singleTool.empty()) { - tools.push_back(singleTool); - } - } - - pos = end; - toolCallPositions.emplace_back(start, end); - } - - for (const std::string& tool : tools) { - ToolCall toolCall; - auto wasToolCallParsed = parseSingleToolCall(tool, toolCall); - if (wasToolCallParsed) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool call - name: {}, args: {}", toolCall.name, toolCall.arguments); - parsedOutput.toolCalls.push_back(toolCall); - } else { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse tool call from string: {}", tool); - } - } - std::vector contentWithoutToolCalls = generatedTokens; - for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { - contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); - } - - auto reasoningEnd = std::find(contentWithoutToolCalls.begin(), contentWithoutToolCalls.end(), reasoningEndTokenId); - if (reasoningEnd != contentWithoutToolCalls.end()) { - contentWithoutToolCalls.erase(contentWithoutToolCalls.begin(), reasoningEnd + 1); - } - parsedOutput.content = tokenizer.decode(contentWithoutToolCalls, ov::AnyMap{ov::genai::skip_special_tokens(true)}); -} } // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp b/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp index 0a7ab12b15..460c5af9c9 100644 --- a/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp +++ b/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp @@ -17,7 +17,10 @@ #include #include #include + #include "src/llm/io_processing/base_output_parser.hpp" +#include "src/port/rapidjson_stringbuffer.hpp" +#include "src/port/rapidjson_writer.hpp" namespace ovms { class Gemma4ToolParser : public BaseOutputParser { @@ -48,33 +51,30 @@ class Gemma4ToolParser : public BaseOutputParser { public: Gemma4ToolParser() = delete; - explicit Gemma4ToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags = {TOOL_CALL_START_TAG}; - return parsingStartTags; - } - const std::vector& getSpecialTagsToErase() const override { - static const std::vector tagsToErase = {TURN_END_TAG, TOOL_RESPONSE_START_TAG}; - return tagsToErase; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"<|tool_call>"}; + cfg.tokenIdStartTags = {"<|tool_call>"}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector beginningOnlyTags = {}; - return beginningOnlyTags; + explicit Gemma4ToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + streamingContent.clear(); + streamingPosition = 0; + currentState = State::Content; + toolCall = {}; + toolCallIndex = -1; } - const std::string& getParsingEndTag() const override { - return TOOL_CALL_END_TAG; - } - - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; static std::string normalizeArgStr(const std::string& arg); static std::string parseArrayParameter(const std::string& argumentStr); @@ -93,8 +93,8 @@ class Gemma4ToolParser : public BaseOutputParser { bool parseToolCallParametersState(); bool parseInToolCallEndedState(); - std::optional wrapDeltaContent(const std::string& content); - rapidjson::Document wrapDeltaArgs(const std::string& argsStr, int toolCallIndex); + std::optional wrapDeltaContent(const std::string& content); + ToolCallDelta wrapDeltaArgs(const std::string& argsStr, int toolCallIndex); std::string streamingContent; size_t streamingPosition{0}; diff --git a/src/llm/io_processing/gptoss/reasoning_parser.cpp b/src/llm/io_processing/gptoss/reasoning_parser.cpp index 2d856cbd85..138b80b53e 100644 --- a/src/llm/io_processing/gptoss/reasoning_parser.cpp +++ b/src/llm/io_processing/gptoss/reasoning_parser.cpp @@ -18,29 +18,14 @@ #include #include -#include "src/port/rapidjson_document.hpp" - #include "../../../logging.hpp" #include "../../../stringutils.hpp" #include "reasoning_parser.hpp" #include "harmony.hpp" -#include "../utils.hpp" namespace ovms { -void GptOssReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - openai::Harmony harmony(tokenizer, generatedTokens); - if (!harmony.parse()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Harmony parsing failed"); - return; - } - - parsedOutput.content = harmony.getContent(); - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | GPT Content | [{}]", parsedOutput.content); - parsedOutput.reasoning = harmony.getReasoning(); - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | GPT Reasoning | [{}]", parsedOutput.reasoning); -} -std::optional GptOssReasoningParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional GptOssReasoningParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Reason | Processing Chunk [{}]", newChunk); if (newChunk.empty()) { @@ -51,26 +36,13 @@ std::optional GptOssReasoningParser::parseChunk(const std:: StreamState lastState = state; - if (startsWith(chunk, getParsingStartTags()[0])) { - // Final content + if (startsWith(chunk, parsingConfig.startTags[0])) { state = StreamState::READING_REASONING; - chunk = chunk.substr(getParsingStartTags()[0].size()); - } else if (startsWith(chunk, "<|start|>assistant<|channel|>final<|message|>")) { - // Final content - state = StreamState::READING_CONTENT; - chunk = chunk.substr(std::strlen("<|start|>assistant<|channel|>final<|message|>")); - } else if (startsWith(chunk, "<|channel|>final<|message|>")) { - // Final content - state = StreamState::READING_CONTENT; - chunk = chunk.substr(std::strlen("<|channel|>final<|message|>")); - } else if (startsWith(chunk, "<|channel|>commentary<|message|>")) { - // Preamble - state = StreamState::READING_CONTENT; - chunk = chunk.substr(std::strlen("<|channel|>commentary<|message|>")); - } else if (endsWith(chunk, getParsingEndTag())) { + chunk = chunk.substr(parsingConfig.startTags[0].size()); + } else if (endsWith(chunk, parsingConfig.endTag)) { // End state = StreamState::UNKNOWN; - chunk = chunk.substr(0, chunk.size() - getParsingEndTag().size()); + chunk = chunk.substr(0, chunk.size() - parsingConfig.endTag.size()); } else if (endsWith(chunk, "<|return|>")) { // End state = StreamState::UNKNOWN; @@ -80,32 +52,9 @@ std::optional GptOssReasoningParser::parseChunk(const std:: if (chunk.size() == 0) return std::nullopt; - switch (lastState) { - case StreamState::READING_REASONING: - case StreamState::READING_CONTENT: { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - if (state == StreamState::READING_REASONING) - writer.String("reasoning_content"); - else - writer.String("content"); - writer.String(chunk.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - - if (state == StreamState::READING_REASONING) - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Reason | Sending Reasoning [{}]", chunk); - else - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Reason | Sending Content [{}]", chunk); - return doc; - } - case StreamState::UNKNOWN: - break; + if (lastState == StreamState::READING_REASONING) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Reason | Sending Reasoning [{}]", chunk); + return ReasoningDelta{chunk}; } return std::nullopt; diff --git a/src/llm/io_processing/gptoss/reasoning_parser.hpp b/src/llm/io_processing/gptoss/reasoning_parser.hpp index 37af80b4bf..af9c133107 100644 --- a/src/llm/io_processing/gptoss/reasoning_parser.hpp +++ b/src/llm/io_processing/gptoss/reasoning_parser.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -26,9 +27,8 @@ namespace ovms { /* - This parser handles reasoning, but is also responsible for parsing regular content. - This model group requires use of reasoning to work even if reasoning is not needed. - This is due to the fact that regular content is placed in harmony format in similar fashion as reasoning. + This parser handles only the analysis (reasoning) channel of the harmony format. + Regular content (final/commentary channels) is handled separately by GptOssContentParser. */ class GptOssReasoningParser : public BaseOutputParser { protected: @@ -38,42 +38,28 @@ class GptOssReasoningParser : public BaseOutputParser { enum class StreamState : int { UNKNOWN = 0, READING_REASONING = 1, - READING_CONTENT = 2, }; StreamState state = StreamState::UNKNOWN; public: GptOssReasoningParser() = delete; - explicit GptOssReasoningParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - // Unary - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - // Streaming - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - - const std::vector& getParsingStartTags() const override { - // If you add another element you have to update implementation as well - // as mostly it assumed just one element - static const std::vector parsingStartTags{parsingStartTag}; - return parsingStartTags; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"<|channel|>analysis<|message|>"}; + cfg.endTag = "<|end|>"; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags = { - "<|channel|>final<|message|>", - "<|channel|>commentary<|message|>", // Preable to reasoning, users usually sees that - "<|start|>assistant<|channel|>final<|message|>", // Final content users sees - }; - return specialParsingStartTags; - } + explicit GptOssReasoningParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} - const std::string& getParsingEndTag() const override { - return parsingEndTag; - } + void resetState() override { state = StreamState::UNKNOWN; } - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/gptoss/tool_parser.cpp b/src/llm/io_processing/gptoss/tool_parser.cpp index 2ca3a50f7b..30f2ceb2d4 100644 --- a/src/llm/io_processing/gptoss/tool_parser.cpp +++ b/src/llm/io_processing/gptoss/tool_parser.cpp @@ -19,8 +19,6 @@ #include #include -#include "src/port/rapidjson_document.hpp" - #include "../../../logging.hpp" #include "../../../stringutils.hpp" #include "tool_parser.hpp" @@ -29,49 +27,12 @@ namespace ovms { -void GptOssToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - openai::Harmony harmony(tokenizer, generatedTokens); - if (!harmony.parse()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Harmony parsing failed"); - return; - } - - // Yes, getContent is called twice, once in reasoning parser and once here, in tool parser. - // This is because we have no guarantee that user will use both parsers, they might use only one of them. - parsedOutput.content = harmony.getContent(); - parsedOutput.toolCalls = harmony.getToolCalls(); - for (const auto& toolCall : parsedOutput.toolCalls) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | GPT Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); - } -} - /* Prepares document with {"arguments": "escaped_chunk"} String gets escaped automatically by rapidjson */ -std::optional GptOssToolParser::wrapDeltaIntoDocument(const std::string& chunk) { - rapidjson::Document newDelta; - newDelta.SetObject(); - rapidjson::Value argumentsValue; - argumentsValue.SetString(chunk.c_str(), static_cast(chunk.size()), newDelta.GetAllocator()); - newDelta.AddMember("arguments", argumentsValue, newDelta.GetAllocator()); - rapidjson::Document wrappedDelta; - wrappedDelta.SetObject(); - rapidjson::Value toolCalls(rapidjson::kArrayType); - rapidjson::Value toolCallObj(rapidjson::kObjectType); - toolCallObj.AddMember("index", toolCallIndex, wrappedDelta.GetAllocator()); - rapidjson::Value functionObj(rapidjson::kObjectType); - for (auto it = newDelta.MemberBegin(); it != newDelta.MemberEnd(); ++it) { - rapidjson::Value key(it->name, wrappedDelta.GetAllocator()); - rapidjson::Value value(it->value, wrappedDelta.GetAllocator()); - functionObj.AddMember(key, value, wrappedDelta.GetAllocator()); - } - toolCallObj.AddMember("function", functionObj, wrappedDelta.GetAllocator()); - toolCalls.PushBack(toolCallObj, wrappedDelta.GetAllocator()); - rapidjson::Value deltaWrapper(rapidjson::kObjectType); - deltaWrapper.AddMember("tool_calls", toolCalls, wrappedDelta.GetAllocator()); - wrappedDelta.AddMember("delta", deltaWrapper, wrappedDelta.GetAllocator()); - return wrappedDelta; +std::optional GptOssToolParser::wrapDeltaIntoDocument(const std::string& chunk) { + return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, chunk}; } void GptOssToolParser::clearState() { @@ -80,13 +41,13 @@ void GptOssToolParser::clearState() { functionNameCache.clear(); } -std::optional GptOssToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional GptOssToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Tool | Processing Chunk [{}]", newChunk); std::string chunk = newChunk; - std::optional result; + std::optional result; - for (const auto& parsingStartTag : getParsingStartTags()) { + for (const auto& parsingStartTag : parsingConfig.startTags) { if (chunk.find(parsingStartTag) != std::string::npos) { toolCallIndex++; // starting with -1, first call will be 0 return std::nullopt; @@ -100,7 +61,7 @@ std::optional GptOssToolParser::parseChunk(const std::strin if (streamState == StreamState::READING_CHANNEL) { if (functionNameCache.size()) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Tool | Sending Function Name [{}]", functionNameCache); - result = wrapFirstDelta(functionNameCache, toolCallIndex); + result = ToolCallDelta{toolCallIndex, generateRandomId(), functionNameCache, ""}; } } else { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Error: <|constrain|> appearance without previous <|channel|>, ignoring"); @@ -119,7 +80,7 @@ std::optional GptOssToolParser::parseChunk(const std::strin if (streamState == StreamState::READING_CHANNEL) { if (functionNameCache.size()) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Streaming | GPT Tool | Sending Function Name [{}]", functionNameCache); - result = wrapFirstDelta(functionNameCache, toolCallIndex); + result = ToolCallDelta{toolCallIndex, generateRandomId(), functionNameCache, ""}; } } @@ -218,5 +179,4 @@ std::optional GptOssToolParser::parseChunk(const std::strin const std::string GptOssToolParser::parsingStartTag = "<|channel|>commentary to="; const std::string GptOssToolParser::parsingEndTag = "<|call|>"; - } // namespace ovms diff --git a/src/llm/io_processing/gptoss/tool_parser.hpp b/src/llm/io_processing/gptoss/tool_parser.hpp index ff6655db37..3063094c63 100644 --- a/src/llm/io_processing/gptoss/tool_parser.hpp +++ b/src/llm/io_processing/gptoss/tool_parser.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -43,39 +44,34 @@ class GptOssToolParser : public BaseOutputParser { int toolCallIndex = -1; std::string functionNameCache; - std::optional wrapDeltaIntoDocument(const std::string& chunk); + std::optional wrapDeltaIntoDocument(const std::string& chunk); void clearState(); public: GptOssToolParser() = delete; - explicit GptOssToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - // Unary - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - // Streaming - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{ - parsingStartTag, - "<|channel|>analysis to=", // Workaround: allow tool calls emitted from the analysis channel (non-standard behavior observed in some model outputs). - }; - return parsingStartTags; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"<|channel|>commentary to=", + "<|channel|>analysis to="}; + cfg.endTag = "<|call|>"; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags = {}; - return specialParsingStartTags; - } + explicit GptOssToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} - const std::string& getParsingEndTag() const override { - return parsingEndTag; + void resetState() override { + streamState = StreamState::READING_CHANNEL; + toolCallIndex = -1; + clearState(); } - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/hermes3/tool_parser.cpp b/src/llm/io_processing/hermes3/tool_parser.cpp index c4b1d55b4a..eecab9f837 100644 --- a/src/llm/io_processing/hermes3/tool_parser.cpp +++ b/src/llm/io_processing/hermes3/tool_parser.cpp @@ -116,69 +116,7 @@ void Hermes3ToolParser::clearState() { argumentsDelayWindow[1].clear(); } -void Hermes3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - const std::string startTag = ""; - const std::string endTag = ""; - std::vector tools; - size_t pos = 0; - size_t firstToolCallPos; - - // Save position of the first tool call start tag to properly clear content after parsing. - firstToolCallPos = parsedOutput.content.find(startTag, pos); - while (true) { - size_t start = parsedOutput.content.find(startTag, pos); - if (start == std::string::npos) { - break; - } - start += startTag.length(); - size_t end = parsedOutput.content.find(endTag, start); - std::string tool; - if (end != std::string::npos) { - tool = parsedOutput.content.substr(start, end - start); - pos = end + endTag.length(); - } else { - tool = parsedOutput.content.substr(start); - pos = parsedOutput.content.length(); - } - if (!tool.empty()) { - tools.push_back(tool); - } - } - - for (const std::string& tool : tools) { - ToolCall toolCall; - rapidjson::Document toolDoc; - toolDoc.Parse(tool.c_str()); - if (toolDoc.HasParseError()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to parse tool call as JSON"); - continue; - } - if (toolDoc.HasMember("name") && toolDoc["name"].IsString()) { - toolCall.name = toolDoc["name"].GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid name field"); - continue; - } - - if (toolDoc.HasMember("arguments") && toolDoc["arguments"].IsObject()) { - rapidjson::StringBuffer sb; - rapidjson::Writer toolWriter(sb); - toolDoc["arguments"].Accept(toolWriter); - toolCall.arguments = sb.GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid parameters object"); - continue; - } - toolCall.id = generateRandomId(); // Generate a random ID for the tool call - parsedOutput.toolCalls.push_back(toolCall); - } - // Remove tool calls from the content - if (firstToolCallPos != std::string::npos) { - parsedOutput.content.erase(firstToolCallPos); - } -} - -std::optional Hermes3ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional Hermes3ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { /* We first collect data until we have full function name - that's when we return the first delta. Every next delta contains next parts of the arguments. Hermes3 generates arguments as JSON, but OpenAI API expects them in a string format. @@ -203,7 +141,15 @@ std::optional Hermes3ToolParser::parseChunk(const std::stri toolCallCompleted = (finishReason != ov::genai::GenerationFinishReason::NONE); - if (chunk.empty()) { + const bool hasPendingState = + !unprocessedBuffer.empty() || + !argumentsDelayWindow[0].empty() || + !argumentsDelayWindow[1].empty() || + lastJson.HasMember("arguments"); + + // Empty chunks are usually ignorable, except finalization calls when we still + // have delayed argument state to flush (e.g. empty STOP chunk from streamer). + if (chunk.empty() && !hasPendingState) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Hermes3ToolParser"); return std::nullopt; } @@ -326,7 +272,6 @@ std::optional Hermes3ToolParser::parseChunk(const std::stri throw std::runtime_error("Generated tool call structure is not valid"); } - rapidjson::Document doc; // Case 1: 'arguments' has just appeared in the current chunk. If so, we return first delta. if (newJson.HasMember("arguments") && !lastJson.HasMember("arguments")) { std::string functionName; @@ -340,9 +285,8 @@ std::optional Hermes3ToolParser::parseChunk(const std::stri throw std::runtime_error("Tool call name is missing in generated output"); } // Wrap first delta in {"tool_calls":[{"id":,"type":"function","index":,"function":{"name": }}]} - doc = wrapFirstDelta(functionName, toolCallIndex); lastJson.CopyFrom(newJson, lastJson.GetAllocator()); - return doc; + return ToolCallDelta{toolCallIndex, generateRandomId(), functionName, ""}; // Case 2: 'arguments' already exists in the last JSON, we compute delta and return it. } else if (lastJson.HasMember("arguments")) { rapidjson::Document delta = PartialJsonBuilder::computeDelta(lastJson, newJson); @@ -356,9 +300,11 @@ std::optional Hermes3ToolParser::parseChunk(const std::stri return std::nullopt; } } - // Wrap delta in {"tool_calls":[{"index":,"function":}]} - doc = wrapDelta(delta, toolCallIndex); - return doc; + // Wrap delta in {"tool_calls":[{"index":,"function":{"arguments":"..."}}]} + std::string argsStr; + if (delta.HasMember("arguments") && delta["arguments"].IsString()) + argsStr = delta["arguments"].GetString(); + return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, argsStr}; // Case 3: No 'arguments' exists or just appeared, so we keep building up until we have complete function name } else { lastJson.CopyFrom(newJson, lastJson.GetAllocator()); diff --git a/src/llm/io_processing/hermes3/tool_parser.hpp b/src/llm/io_processing/hermes3/tool_parser.hpp index dc8f98d634..df5d0d0220 100644 --- a/src/llm/io_processing/hermes3/tool_parser.hpp +++ b/src/llm/io_processing/hermes3/tool_parser.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -70,22 +71,29 @@ class Hermes3ToolParser : public BaseOutputParser { public: Hermes3ToolParser() = delete; - explicit Hermes3ToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags = {parsingStartTag}; - return parsingStartTags; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {""}; + cfg.endTag = ""; + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector beginningOnlyTags = {}; - return beginningOnlyTags; - } - // Tools calls are expected to be the last part of the content, so we do not specify an end tag. - const std::string& getParsingEndTag() const override { - return parsingEndTag; + + explicit Hermes3ToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + lastJson.SetNull(); + jsonBuilder.clear(); + toolCallIndex = -1; + argumentsDelayWindow[0].clear(); + argumentsDelayWindow[1].clear(); + unprocessedBuffer.clear(); + toolCallCompleted = false; } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_reasoning_parser.cpp b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.cpp deleted file mode 100644 index 8d49e384dd..0000000000 --- a/src/llm/io_processing/lfm2/lfm25_reasoning_parser.cpp +++ /dev/null @@ -1,72 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include - -#include "src/port/rapidjson_document.hpp" - -#include "../../../logging.hpp" -#include "lfm25_reasoning_parser.hpp" -#include "../utils.hpp" - -namespace ovms { -void Lfm25ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - auto startReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningStartTokenId); - auto endReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId); - - if (startReasoningIt == generatedTokens.end() || endReasoningIt == generatedTokens.end() || startReasoningIt >= endReasoningIt) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Lfm25ReasoningParser: Reasoning start or end token not found in the generated tokens, or in wrong order. Start token found: {}, End token found: {}, Start position: {}, End position: {}", - startReasoningIt != generatedTokens.end(), endReasoningIt != generatedTokens.end(), std::distance(generatedTokens.begin(), startReasoningIt), std::distance(generatedTokens.begin(), endReasoningIt)); - return; - } - - auto startPos = std::distance(generatedTokens.begin(), startReasoningIt); - auto endPos = std::distance(generatedTokens.begin(), endReasoningIt); - - std::string reasoningContent = tokenizer.decode(std::vector(startPos + generatedTokens.begin() + 1, endPos + generatedTokens.begin()), ov::genai::skip_special_tokens(true)); - - parsedOutput.reasoning = reasoningContent; - - std::string contentWithoutReasoning = tokenizer.decode(std::vector(generatedTokens.begin() + endPos + 1, generatedTokens.end()), ov::genai::skip_special_tokens(true)); // content MUST never appear before reasoning - parsedOutput.content = contentWithoutReasoning; -} - -std::optional Lfm25ReasoningParser::parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) { - if (tokens.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty tokens for Lfm25ReasoningParser"); - return std::nullopt; - } - - if (std::find(tokens.begin(), tokens.end(), reasoningStartTokenId) != tokens.end() || - std::find(tokens.begin(), tokens.end(), reasoningEndTokenId) != tokens.end()) { - return std::nullopt; - } else { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("reasoning_content"); - writer.String(chunk.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - return doc; - } -} -} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp index afc52c7f56..1ca4305c60 100644 --- a/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp +++ b/src/llm/io_processing/lfm2/lfm25_reasoning_parser.hpp @@ -14,41 +14,33 @@ // limitations under the License. //***************************************************************************** #pragma once -#include "../base_output_parser.hpp" -#include +#include "../qwen3/reasoning_parser.hpp" +#include #include +#include +#include namespace ovms { -class Lfm25ReasoningParser : public BaseOutputParser { -protected: - const std::string parsingStartTag = ""; - const std::string parsingEndTag = ""; - - const int64_t reasoningStartTokenId = 124901; // - const int64_t reasoningEndTokenId = 124902; // - +// LFM2.5 reasoning uses the same /<\think> grammar as Qwen3 but both delimiters +// are registered special tokens (not regular vocabulary), so tokenIdStartTags is set and +// needsSpecialTokens/defaultDecodingWithSpecialTokens are both true. +class Lfm25ReasoningParser : public Qwen3ReasoningParser { public: Lfm25ReasoningParser() = delete; - explicit Lfm25ReasoningParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{this->parsingStartTag}; - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - return parsingEndTag; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {""}; + cfg.tokenIdStartTags = {""}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - // It may be removed after changing logic in Lfm2ToolParser to use tokens in streaming instead of chunk content, both tool parser and reasoning parser need to have the same value for this function - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + explicit Lfm25ReasoningParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + Qwen3ReasoningParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} }; } // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_tool_parser.cpp b/src/llm/io_processing/lfm2/lfm25_tool_parser.cpp deleted file mode 100644 index d651ed359f..0000000000 --- a/src/llm/io_processing/lfm2/lfm25_tool_parser.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "lfm25_tool_parser.hpp" - -namespace ovms { - -const std::string Lfm25ToolParser::TOOL_CALL_START_TAG = "<|tool_call_start|>"; -const std::string Lfm25ToolParser::TOOL_CALL_END_TAG = "<|tool_call_end|>"; - -const int64_t Lfm25ToolParser::toolCallStartTokenId = 124905; // <|tool_call_start|> -const int64_t Lfm25ToolParser::toolCallEndTokenId = 124906; // <|tool_call_end|> -const int64_t Lfm25ToolParser::reasoningStartTokenId = 124901; // -const int64_t Lfm25ToolParser::reasoningEndTokenId = 124902; // - -bool Lfm25ToolParser::parseNewContent() { - switch (this->currentState) { - case State::Content: { - return parseInContentState(this->streamingContent, this->streamingPosition, this->currentState, this->tagIds); - } - case State::ToolCallStarted: { - auto wasParsedCorrectly = parseInToolCallState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); - if (wasParsedCorrectly) { - this->toolCallIndex++; - } - return wasParsedCorrectly; - } - case State::ToolCallParameters: { - return parseInToolCallParametersState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); - } - case State::ToolCallEnded: { - return parseInToolCallEndedState(this->streamingContent, this->streamingPosition, this->currentState, TOOL_CALL_END_TAG); - } - case State::AfterToolCall: - break; - } - return false; -} - -std::optional Lfm25ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { - if (chunk.empty()) { - return std::nullopt; - } - - this->streamingContent += chunk; - - if (parseNewContent()) { - if (this->currentState == State::ToolCallParameters) { - return BaseOutputParser::wrapFirstDelta(this->toolCall.name, this->toolCallIndex); - } - if (this->currentState == State::ToolCallEnded) { - return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); - } - if (this->currentState == State::Content) { - size_t contentEnd = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); - std::string content; - if (contentEnd != std::string::npos) { - content = this->streamingContent.substr(this->streamingPosition, contentEnd - this->streamingPosition); - } else { - content = this->streamingContent.substr(this->streamingPosition); - } - this->streamingPosition += content.size(); - cutEOSFromContent(content); - - if (!content.empty()) { - return wrapDeltaContent(content); - } - } - if (this->currentState == State::AfterToolCall) { - this->currentState = State::Content; - } - } - - if (finishReason != ov::genai::GenerationFinishReason::NONE) { - if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { - return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); - } - - if (this->currentState == State::Content && this->streamingPosition < this->streamingContent.size()) { - auto content = this->streamingContent.substr(this->streamingPosition); - this->streamingPosition += content.size(); - cutEOSFromContent(content); - - if (!content.empty()) { - return wrapDeltaContent(content); - } - } - } - - return std::nullopt; -} - -void Lfm25ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - parseUnaryResponse(parsedOutput, generatedTokens, tokenizer, this->tagIds); -} -} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm25_tool_parser.hpp b/src/llm/io_processing/lfm2/lfm25_tool_parser.hpp deleted file mode 100644 index abcf288dcb..0000000000 --- a/src/llm/io_processing/lfm2/lfm25_tool_parser.hpp +++ /dev/null @@ -1,72 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include "lfm2_utils.hpp" - -namespace ovms { -class Lfm25ToolParser : public BaseOutputParser { -public: - static const std::string TOOL_CALL_START_TAG; - static const std::string TOOL_CALL_END_TAG; - - static const int64_t toolCallStartTokenId; - static const int64_t toolCallEndTokenId; - static const int64_t reasoningStartTokenId; - static const int64_t reasoningEndTokenId; - - Lfm25ToolParser() = delete; - explicit Lfm25ToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags = {TOOL_CALL_START_TAG}; - return parsingStartTags; - } - - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector beginningOnlyTags = {}; - return beginningOnlyTags; - } - - const std::vector& getSpecialTagsToErase() const override { - static const std::vector tagsToErase = {EOS_TOKEN_STR}; - return tagsToErase; - } - - const std::string& getParsingEndTag() const override { - return TOOL_CALL_END_TAG; - } - - bool requiresStreamingWithSpecialTokens() const override { - return true; - } - -private: - std::string streamingContent; - size_t streamingPosition{0}; - State currentState{State::Content}; - ToolCall toolCall; - TagIds tagIds{TOOL_CALL_START_TAG, TOOL_CALL_END_TAG, toolCallStartTokenId, toolCallEndTokenId, reasoningStartTokenId, reasoningEndTokenId}; - - int toolCallIndex{TOOL_CALL_INDEX_START}; - - bool parseNewContent(); -}; -} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp b/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp index 3e13a2b681..df15b95720 100644 --- a/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp +++ b/src/llm/io_processing/lfm2/lfm2_tool_parser.cpp @@ -14,93 +14,366 @@ // limitations under the License. //***************************************************************************** #include "lfm2_tool_parser.hpp" +#include "../utils.hpp" +#include "../../../logging.hpp" +#include "../../../stringutils.hpp" +#include "src/port/rapidjson_document.hpp" +#include "rapidjson/error/en.h" + +#include +#include +#include namespace ovms { -const std::string Lfm2ToolParser::TOOL_CALL_START_TAG = "<|tool_call_start|>"; -const std::string Lfm2ToolParser::TOOL_CALL_END_TAG = "<|tool_call_end|>"; +namespace { -const int64_t Lfm2ToolParser::toolCallStartTokenId = 10; // <|tool_call_start|> -const int64_t Lfm2ToolParser::toolCallEndTokenId = 11; // <|tool_call_end|> +// LFM2.5 assigns token ID 124905 to <|tool_call_start|>; LFM2 uses 10. +// (Token-ID resolution happens automatically via tokenIdStartTags.) -bool Lfm2ToolParser::parseNewContent() { - switch (this->currentState) { - case State::Content: { - return parseInContentState(this->streamingContent, this->streamingPosition, this->currentState, this->tagIds); +// Tool-call format delimiters shared by LFM2 and LFM2.5. +const std::string TOOL_LIST_START_INDICATOR = "["; +const std::string TOOL_LIST_END_INDICATOR = "]"; +const std::string TOOL_ARGS_START_INDICATOR = "("; +const std::string TOOL_ARGS_END_INDICATOR = ")"; +const std::string TOOL_SEPARATOR_STR = ", "; +// EOS token emitted by the LFM2.5 chat template after tool-call blocks. +const std::string EOS_TOKEN_STR = "<|im_end|>"; + +struct Argument { + std::string name; + std::string value; +}; + +// --------------------------------------------------------------------------- +// Argument-value normalisation helpers +// --------------------------------------------------------------------------- + +std::string parseArrayParameter(std::string argumentStr) { + int quoteDepth = 0; + for (size_t i = 1; i < argumentStr.size() - 1; ++i) { + if (argumentStr[i] != '\'') + continue; + bool isLastElement = (i == argumentStr.size() - 2); + bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; + if (quoteDepth == 0) { + argumentStr[i] = '"'; + quoteDepth++; + } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement)) { + argumentStr[i] = '"'; + quoteDepth--; + } } - case State::ToolCallStarted: { - auto wasParsedCorrectly = parseInToolCallState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); - if (wasParsedCorrectly) { - this->toolCallIndex++; + return argumentStr; +} + +std::string parseObjectParameter(std::string argumentStr) { + int quoteDepth = 0; + for (size_t i = 1; i < argumentStr.size() - 1; ++i) { + if (argumentStr[i] != '\'') + continue; + bool isLastElement = (i == argumentStr.size() - 2); + bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; + bool isFollowedByColon = !isLastElement && argumentStr[i + 1] == ':'; + if (quoteDepth == 0) { + argumentStr[i] = '"'; + quoteDepth++; + } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement || isFollowedByColon)) { + argumentStr[i] = '"'; + quoteDepth--; + } + } + return argumentStr; +} + +std::string normalizeArgStr(const std::string& arg) { + if (arg.empty()) + return arg; + + std::string normalized = arg; + trim(normalized); + std::string lower = normalized; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + + if (lower == "true" || lower == "false" || lower == "null") + return lower; + + const char first = normalized.front(); + const char last = normalized.back(); + if (first == '{' && last == '}') { + normalized = parseObjectParameter(normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an object, replaced single quotes: {}", normalized); + } + if (first == '[' && last == ']') { + normalized = parseArrayParameter(normalized); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalised quotes: {}", normalized); + } + if (first == '\'' && last == '\'') { + normalized[0] = '"'; + normalized[normalized.size() - 1] = '"'; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument enclosed in single quotes, replaced with double quotes: {}", normalized); + } + + rapidjson::Document tempDoc; + rapidjson::Value finalValue; + tempDoc.Parse(normalized.c_str()); + if (tempDoc.HasParseError()) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument not valid JSON ({}), treating as string: {}", + rapidjson::GetParseError_En(tempDoc.GetParseError()), normalized); + if (first == '"' && last == '"') + normalized = normalized.substr(1, normalized.size() - 2); + finalValue.SetString(normalized.c_str(), static_cast(normalized.size()), tempDoc.GetAllocator()); + } else { + finalValue.CopyFrom(tempDoc, tempDoc.GetAllocator()); + } + + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + finalValue.Accept(writer); + return buffer.GetString(); +} + +void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { + std::string normalized = normalizeArgStr(arg); + rapidjson::Document doc; + doc.Parse(normalized.c_str()); + rapidjson::Value& argumentDoc = doc; + writeArgumentOfAnyType(argumentDoc, writer); +} + +Argument parseSingleArgument(const std::string& argumentStr) { + Argument argument; + size_t equalPos = argumentStr.find('='); + if (equalPos != std::string::npos) { + argument.name = argumentStr.substr(0, equalPos); + argument.value = argumentStr.substr(equalPos + 1); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.name, argument.value); + } else { + argument.name = argumentStr; + argument.value = ""; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument '{}' has no '='; value set to empty", argumentStr); + } + return argument; +} + +std::vector parseArguments(const std::string& argumentsStr) { + std::vector parsedArgs; + size_t argPos = 0; + while (argPos < argumentsStr.length()) { + size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_SEPARATOR_STR, argPos); + if (commaPos == std::string::npos) { + parsedArgs.push_back(parseSingleArgument(argumentsStr.substr(argPos))); + break; + } + parsedArgs.push_back(parseSingleArgument(argumentsStr.substr(argPos, commaPos - argPos))); + argPos = commaPos + TOOL_SEPARATOR_STR.length(); + } + return parsedArgs; +} + +// --------------------------------------------------------------------------- +// State-machine step functions +// --------------------------------------------------------------------------- + +bool parseInContentState(const std::string& streamingContent, size_t& streamingPosition, + Lfm2ParseState& currentState, + const std::string& startTag, const std::string& endTag) { + size_t startTagPos = streamingContent.find(startTag, streamingPosition); + size_t endTagPos = streamingContent.find(endTag, streamingPosition); + if (endTagPos != std::string::npos && startTagPos == std::string::npos) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected stray end tag at position: {}", endTagPos); + streamingPosition = endTagPos + endTag.length(); + return false; + } + if (startTagPos != std::string::npos) { + if (startTagPos > streamingPosition) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content before tool-call start tag at position: {}", startTagPos); + return true; } - return wasParsedCorrectly; + currentState = Lfm2ParseState::ToolCallStarted; + streamingPosition = startTagPos + startTag.length(); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected tool-call start at position: {}", startTagPos); + return false; } - case State::ToolCallParameters: { - return parseInToolCallParametersState(this->streamingContent, this->toolCall, this->streamingPosition, this->currentState); + return true; +} + +bool parseInToolCallState(const std::string& streamingContent, ToolCall& toolCall, + size_t& streamingPosition, Lfm2ParseState& currentState) { + size_t toolListStartPos = streamingContent.find(TOOL_LIST_START_INDICATOR, streamingPosition); + size_t argsPos = streamingContent.find(TOOL_ARGS_START_INDICATOR, streamingPosition); + + if (toolListStartPos != std::string::npos) { + streamingPosition = toolListStartPos + TOOL_LIST_START_INDICATOR.length(); + } else if (argsPos != std::string::npos) { + size_t bracketAnyPos = streamingContent.find(TOOL_LIST_START_INDICATOR); + if (bracketAnyPos == std::string::npos || bracketAnyPos >= argsPos) + return false; } - case State::ToolCallEnded: { - return parseInToolCallEndedState(this->streamingContent, this->streamingPosition, this->currentState, TOOL_CALL_END_TAG); + + if (argsPos == std::string::npos) + return false; + + std::string toolName = streamingContent.substr(streamingPosition, argsPos - streamingPosition); + trim(toolName); + toolCall = ToolCall{generateRandomId(), toolName, ""}; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); + streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); + currentState = Lfm2ParseState::ToolCallParameters; + return true; +} + +bool parseInToolCallParametersState(const std::string& streamingContent, ToolCall& toolCall, + size_t& streamingPosition, Lfm2ParseState& currentState) { + size_t pos = findInStringRespectingSpecialChars(streamingContent, TOOL_ARGS_END_INDICATOR, streamingPosition); + if (pos == std::string::npos) + return false; + + std::string argumentsStr = streamingContent.substr(streamingPosition, pos - streamingPosition); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed arguments string: {}", argumentsStr); + std::vector arguments = parseArguments(argumentsStr); + + rapidjson::StringBuffer sb; + rapidjson::Writer argsWriter(sb); + argsWriter.StartObject(); + for (const Argument& argument : arguments) { + argsWriter.Key(argument.name.c_str()); + writeArgumentToWriter(argument.value, argsWriter); + } + argsWriter.EndObject(); + toolCall.arguments = sb.GetString(); + currentState = Lfm2ParseState::ToolCallEnded; + streamingPosition = pos + TOOL_ARGS_END_INDICATOR.length(); + return true; +} + +bool parseInToolCallEndedState(const std::string& streamingContent, size_t& streamingPosition, + Lfm2ParseState& currentState, const std::string& endTag) { + size_t listEndPos = streamingContent.find(TOOL_LIST_END_INDICATOR, streamingPosition); + size_t separatorPos = streamingContent.find(TOOL_SEPARATOR_STR, streamingPosition); + size_t endTagPos = streamingContent.find(endTag, streamingPosition); + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "ToolCallEnded: content from pos {}: {}", + streamingPosition, streamingContent.substr(streamingPosition)); + if (listEndPos == std::string::npos && separatorPos == std::string::npos && endTagPos == std::string::npos) + return false; + if (separatorPos != std::string::npos && separatorPos < listEndPos) { + streamingPosition = separatorPos + TOOL_SEPARATOR_STR.length(); + currentState = Lfm2ParseState::ToolCallStarted; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool-call separator at {}, expecting next call", separatorPos); + } else if (endTagPos != std::string::npos) { + streamingPosition = endTagPos + endTag.length(); + currentState = Lfm2ParseState::AfterToolCall; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "End tag at {}", endTagPos); + } else { + streamingPosition = listEndPos + TOOL_LIST_END_INDICATOR.length(); + currentState = Lfm2ParseState::AfterToolCall; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "End of tool list at {}", listEndPos); + } + return true; +} + +// --------------------------------------------------------------------------- +// Delta-wrapping helpers +// --------------------------------------------------------------------------- + +ContentDelta wrapDeltaContent(const std::string& content) { + return ContentDelta{content}; +} + +ToolCallDelta wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { + return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, argsStr}; +} + +void cutEOSFromContent(std::string& content) { + size_t pos = content.find(EOS_TOKEN_STR); + if (pos != std::string::npos) + content = content.substr(0, pos); +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Lfm2ToolParser implementation +// --------------------------------------------------------------------------- + +bool Lfm2ToolParser::parseNewContent() { + const std::string& startTag = parsingConfig.startTags[0]; + const std::string& endTag = parsingConfig.endTag; + switch (this->currentState) { + case Lfm2ParseState::Content: + return parseInContentState(this->streamingContent, this->streamingPosition, + this->currentState, startTag, endTag); + case Lfm2ParseState::ToolCallStarted: { + auto ok = parseInToolCallState(this->streamingContent, this->toolCall, + this->streamingPosition, this->currentState); + if (ok) + this->toolCallIndex++; + return ok; } - case State::AfterToolCall: + case Lfm2ParseState::ToolCallParameters: + return parseInToolCallParametersState(this->streamingContent, this->toolCall, + this->streamingPosition, this->currentState); + case Lfm2ParseState::ToolCallEnded: + return parseInToolCallEndedState(this->streamingContent, this->streamingPosition, + this->currentState, endTag); + case Lfm2ParseState::AfterToolCall: break; } return false; } -std::optional Lfm2ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { - if (chunk.empty()) { +std::optional Lfm2ToolParser::parseChunk(const std::string& chunk, + const std::vector& /*tokens*/, + ov::genai::GenerationFinishReason finishReason) { + // Empty chunks may arrive from the two-step streamer end() (NONE + empty STOP). + // Skip them unless we have buffered state that still needs to be flushed. + const bool hasPendingState = (this->currentState == Lfm2ParseState::ToolCallParameters) || + (this->currentState == Lfm2ParseState::ToolCallEnded); + if (chunk.empty() && !hasPendingState) return std::nullopt; - } this->streamingContent += chunk; if (parseNewContent()) { - if (this->currentState == State::ToolCallParameters) { - return BaseOutputParser::wrapFirstDelta(this->toolCall.name, this->toolCallIndex); + if (this->currentState == Lfm2ParseState::ToolCallParameters) { + return ToolCallDelta{this->toolCallIndex, generateRandomId(), this->toolCall.name, ""}; } - if (this->currentState == State::ToolCallEnded) { + if (this->currentState == Lfm2ParseState::ToolCallEnded) { return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); } - if (this->currentState == State::Content) { - size_t contentEnd = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); - std::string content; - if (contentEnd != std::string::npos) { - content = this->streamingContent.substr(this->streamingPosition, contentEnd - this->streamingPosition); - } else { - content = this->streamingContent.substr(this->streamingPosition); - } + if (this->currentState == Lfm2ParseState::Content) { + const std::string& startTag = parsingConfig.startTags[0]; + size_t contentEnd = this->streamingContent.find(startTag, this->streamingPosition); + std::string content = (contentEnd != std::string::npos) + ? this->streamingContent.substr(this->streamingPosition, contentEnd - this->streamingPosition) + : this->streamingContent.substr(this->streamingPosition); this->streamingPosition += content.size(); cutEOSFromContent(content); - - if (!content.empty()) { + if (!content.empty()) return wrapDeltaContent(content); - } } - if (this->currentState == State::AfterToolCall) { - this->currentState = State::Content; + if (this->currentState == Lfm2ParseState::AfterToolCall) { + this->currentState = Lfm2ParseState::Content; } } if (finishReason != ov::genai::GenerationFinishReason::NONE) { - if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { + if ((this->currentState == Lfm2ParseState::ToolCallParameters || + this->currentState == Lfm2ParseState::ToolCallEnded) && + !this->toolCall.arguments.empty()) { return wrapDeltaArgs(this->toolCall.arguments, this->toolCallIndex); } - - if (this->currentState == State::Content && this->streamingPosition < this->streamingContent.size()) { + if (this->currentState == Lfm2ParseState::Content && + this->streamingPosition < this->streamingContent.size()) { auto content = this->streamingContent.substr(this->streamingPosition); this->streamingPosition += content.size(); cutEOSFromContent(content); - - if (!content.empty()) { + if (!content.empty()) return wrapDeltaContent(content); - } } } return std::nullopt; } -void Lfm2ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - parseUnaryResponse(parsedOutput, generatedTokens, tokenizer, this->tagIds); -} } // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp b/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp index 228616d33d..e0b5b449cb 100644 --- a/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp +++ b/src/llm/io_processing/lfm2/lfm2_tool_parser.hpp @@ -14,60 +14,73 @@ // limitations under the License. //***************************************************************************** #pragma once +#include #include +#include #include + +#include + #include "src/llm/io_processing/base_output_parser.hpp" -#include "../../../logging.hpp" -#include "./lfm2_utils.hpp" namespace ovms { -class Lfm2ToolParser : public BaseOutputParser { -protected: - static const std::string TOOL_CALL_START_TAG; - static const std::string TOOL_CALL_END_TAG; - static const int64_t toolCallStartTokenId; - static const int64_t toolCallEndTokenId; +// Streaming state machine states for LFM2 / LFM2.5 tool-call parsing. +enum class Lfm2ParseState { + Content, + ToolCallStarted, + ToolCallParameters, + ToolCallEnded, + AfterToolCall +}; +// Unified tool parser for both LFM2 and LFM2.5. +// The two model families share identical tool-call grammar; the only differences +// are the token IDs assigned by their respective tokenizers and whether the +// chat template appends <|im_end|> after tool calls (LFM2.5 only). +// The correct OutputParsingConfig variant is chosen automatically via configForTokenizer(). +class Lfm2ToolParser : public BaseOutputParser { public: Lfm2ToolParser() = delete; - explicit Lfm2ToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags = {TOOL_CALL_START_TAG}; - return parsingStartTags; + // OutputParsingConfig for LFM2 and LFM2.5. Both model families use the same + // tool-call grammar and token-boundary strings; the only model-specific + // behaviour (stripping <|im_end|> from content) is a no-op on LFM2 since + // that model's chat template never emits <|im_end|> in tool-call context. + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"<|tool_call_start|>"}; + cfg.tokenIdStartTags = {"<|tool_call_start|>"}; + cfg.endTag = "<|tool_call_end|>"; + cfg.needsSpecialTokens = true; + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector beginningOnlyTags = {}; - return beginningOnlyTags; - } + explicit Lfm2ToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} - const std::vector& getSpecialTagsToErase() const override { - static const std::vector tagsToErase = {EOS_TOKEN_STR}; - return tagsToErase; + void resetState() override { + streamingContent.clear(); + streamingPosition = 0; + currentState = Lfm2ParseState::Content; + toolCall = {}; + toolCallIndex = -1; } - const std::string& getParsingEndTag() const override { - return TOOL_CALL_END_TAG; - } - - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + std::optional parseChunk(const std::string& chunk, + const std::vector& tokens, + ov::genai::GenerationFinishReason finishReason) override; private: std::string streamingContent; size_t streamingPosition{0}; - State currentState{State::Content}; + Lfm2ParseState currentState{Lfm2ParseState::Content}; ToolCall toolCall; - TagIds tagIds{TOOL_CALL_START_TAG, TOOL_CALL_END_TAG, toolCallStartTokenId, toolCallEndTokenId}; - - int toolCallIndex{TOOL_CALL_INDEX_START}; + int toolCallIndex{-1}; bool parseNewContent(); }; + } // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_utils.cpp b/src/llm/io_processing/lfm2/lfm2_utils.cpp deleted file mode 100644 index 331d442a45..0000000000 --- a/src/llm/io_processing/lfm2/lfm2_utils.cpp +++ /dev/null @@ -1,412 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "lfm2_utils.hpp" -#include "../utils.hpp" -#include "../../../logging.hpp" -#include "../../../stringutils.hpp" -#include "rapidjson/error/en.h" -#include -#include -#include - -namespace ovms { -const std::string TOOL_LIST_START_INDICATOR = "["; -const std::string TOOL_LIST_END_INDICATOR = "]"; -const std::string TOOL_ARGS_START_INDICATOR = "("; -const std::string TOOL_ARGS_END_INDICATOR = ")"; -const std::string TOOL_SEPARATOR_STR = ", "; -const std::string EOS_TOKEN_STR = "<|im_end|>"; - -const int TOOL_CALL_INDEX_START = -1; - -std::string parseArrayParameter(std::string argumentStr) { - int quoteDepth = 0; - - for (size_t i = 1; i < argumentStr.size() - 1; ++i) { - if (argumentStr[i] != '\'') { - continue; - } - - bool isLastElement = (i == argumentStr.size() - 2); - bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; - - if (quoteDepth == 0) { - argumentStr[i] = '"'; - quoteDepth++; - } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement)) { - argumentStr[i] = '"'; - quoteDepth--; - } - } - - return argumentStr; -} - -std::string parseObjectParameter(std::string argumentStr) { - int quoteDepth = 0; - - for (size_t i = 1; i < argumentStr.size() - 1; ++i) { - if (argumentStr[i] != '\'') { - continue; - } - - bool isLastElement = (i == argumentStr.size() - 2); - bool isFollowedByComma = !isLastElement && argumentStr[i + 1] == ','; - bool isFollowedByColon = !isLastElement && argumentStr[i + 1] == ':'; - - if (quoteDepth == 0) { - argumentStr[i] = '"'; - quoteDepth++; - } else if (quoteDepth > 0 && (isFollowedByComma || isLastElement || isFollowedByColon)) { - argumentStr[i] = '"'; - quoteDepth--; - } - } - - return argumentStr; -} - -std::string normalizeArgStr(const std::string& arg) { - if (arg.empty()) { - return arg; - } - - std::string normalized = arg; - trim(normalized); - std::string lower = normalized; - std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); - - if (lower == "true" || lower == "false" || lower == "null") { - return lower; - } - - const char first = normalized.front(); - const char last = normalized.back(); - if (first == '{' && last == '}') { - normalized = parseObjectParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument contains is an object, replaced single quotes with double quotes for JSON parsing. Modified string: {}", normalized); - } - - if (first == '[' && last == ']') { - normalized = parseArrayParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, normalized quotes for JSON parsing. Modified string: {}", normalized); - } - - if ((first == '\'' && last == '\'')) { - normalized[0] = '"'; - normalized[normalized.size() - 1] = '"'; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is enclosed in quotes, replaced outer quotes with double quotes for JSON parsing. Modified string: {}", normalized); - } - - rapidjson::Document tempDoc; - rapidjson::Value finalValue; - tempDoc.Parse(normalized.c_str()); - if (tempDoc.HasParseError()) { - auto errorCode = tempDoc.GetParseError(); - auto errorMessage = rapidjson::GetParseError_En(errorCode); - size_t errorOffset = tempDoc.GetErrorOffset(); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse argument string as JSON. Argument string: {}, Error: {} Offset: {}", normalized, errorMessage, errorOffset); - - if (first == '\"' && last == '\"') { - normalized = normalized.substr(1, normalized.size() - 2); - } - finalValue.SetString(normalized.c_str(), static_cast(normalized.size()), tempDoc.GetAllocator()); - } else { - finalValue.CopyFrom(tempDoc, tempDoc.GetAllocator()); - } - - { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - finalValue.Accept(writer); - normalized = buffer.GetString(); - } - - return normalized; -} - -void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { - std::string normalized = normalizeArgStr(arg); - - rapidjson::Document doc; - doc.Parse(normalized.c_str()); - - rapidjson::Value& argumentDoc = doc; - writeArgumentOfAnyType(argumentDoc, writer); -} - -Argument parseSingleArgument(const std::string& argumentStr) { - Argument argument; - - size_t equalPos = argumentStr.find('='); - if (equalPos != std::string::npos) { - argument.name = argumentStr.substr(0, equalPos); - argument.value = argumentStr.substr(equalPos + 1); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.name, argument.value); - } else { - argument.name = argumentStr; - argument.value = ""; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument string: {} does not contain '=', setting name as entire string and value as empty", argumentStr); - } - return argument; -} - -std::vector parseArguments(const std::string& argumentsStr) { - std::vector args; - std::vector parsedArgs; - - size_t argPos = 0; - while (argPos < argumentsStr.length()) { - size_t commaPos = findInStringRespectingSpecialChars(argumentsStr, TOOL_SEPARATOR_STR, argPos); - if (commaPos == std::string::npos) { - auto remainingStr = argumentsStr.substr(argPos); - args.push_back(remainingStr); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more commas found, adding remaining argument string: {}", remainingStr); - break; - } - auto argStr = argumentsStr.substr(argPos, commaPos - argPos); - args.push_back(argStr); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); - argPos = commaPos + TOOL_SEPARATOR_STR.length(); - } - - for (const std::string& arg : args) { - parsedArgs.push_back(parseSingleArgument(arg)); - } - return parsedArgs; -} - -bool parseInContentState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const TagIds& tagIds) { - size_t toolCallStartTagPos = streamingContent.find(tagIds.toolCallStartTag, streamingPosition); - size_t toolCallEndTagPos = streamingContent.find(tagIds.toolCallEndTag, streamingPosition); - if (toolCallEndTagPos != std::string::npos && toolCallStartTagPos == std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); - streamingPosition = toolCallEndTagPos + tagIds.toolCallEndTag.length(); - return false; - } - if (toolCallStartTagPos != std::string::npos) { - if (toolCallStartTagPos > streamingPosition) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content found before tool call start tag at position: {}", toolCallStartTagPos); - return true; - } - currentState = State::ToolCallStarted; - streamingPosition = toolCallStartTagPos + tagIds.toolCallStartTag.length(); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected start of tool call at position: {}", toolCallStartTagPos); - return false; - } - - return true; -} - -bool parseInToolCallState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState) { - size_t toolListStartPos = streamingContent.find(TOOL_LIST_START_INDICATOR, streamingPosition); - size_t argsPos = streamingContent.find(TOOL_ARGS_START_INDICATOR, streamingPosition); - - if (toolListStartPos != std::string::npos) { - streamingPosition = toolListStartPos + TOOL_LIST_START_INDICATOR.length(); - } - - if (argsPos == std::string::npos) { - return false; - } - - std::string toolName = streamingContent.substr(streamingPosition, argsPos - streamingPosition); - trim(toolName); - toolCall = ToolCall{generateRandomId(), toolName, ""}; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); - streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); - currentState = State::ToolCallParameters; - return true; -} - -bool parseInToolCallParametersState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState) { - size_t pos = findInStringRespectingSpecialChars(streamingContent, TOOL_ARGS_END_INDICATOR, streamingPosition); - if (pos == std::string::npos) { - return false; - } - std::string argumentsStr = streamingContent.substr(streamingPosition, pos - streamingPosition); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed arguments string: {}", argumentsStr); - std::vector arguments = parseArguments(argumentsStr); - - rapidjson::Document argsDoc(rapidjson::kObjectType); - rapidjson::StringBuffer sb; - rapidjson::Writer argsWriter(sb); - argsWriter.StartObject(); - - for (const Argument& argument : arguments) { - argsWriter.Key(argument.name.c_str()); - writeArgumentToWriter(argument.value, argsWriter); - } - - argsWriter.EndObject(); - toolCall.arguments = sb.GetString(); - currentState = State::ToolCallEnded; - streamingPosition = pos + TOOL_ARGS_END_INDICATOR.length(); - - return true; -} - -bool parseInToolCallEndedState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const std::string& toolCallEndTag) { - size_t pos = streamingContent.find(TOOL_LIST_END_INDICATOR, streamingPosition); - size_t toolSeparatorPos = streamingContent.find(TOOL_SEPARATOR_STR, streamingPosition); - size_t toolCallEndTagPos = streamingContent.find(toolCallEndTag, streamingPosition); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", streamingContent.substr(streamingPosition)); - if (pos == std::string::npos && toolSeparatorPos == std::string::npos && toolCallEndTagPos == std::string::npos) { - return false; - } else if (toolSeparatorPos != std::string::npos && toolSeparatorPos < pos) { - streamingPosition = toolSeparatorPos + TOOL_SEPARATOR_STR.length(); - currentState = State::ToolCallStarted; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected separator between tool calls at position: {}, expecting another tool call to start", toolSeparatorPos); - } else if (toolCallEndTagPos != std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); - streamingPosition = toolCallEndTagPos + toolCallEndTag.length(); - currentState = State::AfterToolCall; - } else { - streamingPosition = pos + TOOL_LIST_END_INDICATOR.length(); - currentState = State::AfterToolCall; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool list at position: {}, returning to content state", pos); - } - return true; -} - -rapidjson::Document wrapDeltaContent(const std::string& content) { - rapidjson::Document doc(rapidjson::kObjectType); - rapidjson::Value deltaObj(rapidjson::kObjectType); - deltaObj.AddMember("content", rapidjson::Value(content.c_str(), doc.GetAllocator()), doc.GetAllocator()); - doc.AddMember("delta", deltaObj, doc.GetAllocator()); - return doc; -} - -rapidjson::Document wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { - rapidjson::Document doc(rapidjson::kObjectType); - doc.AddMember("arguments", rapidjson::Value(argsStr.c_str(), doc.GetAllocator()), doc.GetAllocator()); - - return BaseOutputParser::wrapDelta(doc, toolCallIndex); -} - -void cutEOSFromContent(std::string& content) { - size_t eosPos = content.find(EOS_TOKEN_STR); - if (eosPos != std::string::npos) { - content = content.substr(0, eosPos); - } -} - -bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall) { - size_t argsPos = toolStr.find(TOOL_ARGS_START_INDICATOR); - if (argsPos != std::string::npos) { - std::string toolName = toolStr.substr(0, argsPos); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); - - int argsStrLen = toolStr.length() - argsPos - TOOL_ARGS_START_INDICATOR.length() - TOOL_ARGS_END_INDICATOR.length(); - std::string argsStr = toolStr.substr(argsPos + TOOL_ARGS_START_INDICATOR.length(), argsStrLen); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed args string: {}", argsStr); - std::vector arguments = parseArguments(argsStr); - - toolCall.name = toolName; - rapidjson::Document argsDoc(rapidjson::kObjectType); - rapidjson::StringBuffer sb; - rapidjson::Writer argsWriter(sb); - argsWriter.StartObject(); - for (const Argument& argument : arguments) { - argsWriter.Key(argument.name.c_str()); - writeArgumentToWriter(argument.value, argsWriter); - } - argsWriter.EndObject(); - toolCall.arguments = sb.GetString(); - toolCall.id = generateRandomId(); - return true; - } - return false; -} - -void parseUnaryResponse(ParsedOutput& parsedOutput, const std::vector& generatedTokens, ov::genai::Tokenizer& tokenizer, const TagIds& tagIds) { - std::vector tools; - std::vector> toolCallPositions; - size_t pos = 0; - - while (pos != std::string::npos) { - size_t start, end; - auto it = std::find(generatedTokens.begin() + pos, generatedTokens.end(), tagIds.toolCallStartTokenId); - if (it != generatedTokens.end()) { - start = std::distance(generatedTokens.begin(), it); - } else { - break; - } - auto itArgs = std::find(generatedTokens.begin() + start, generatedTokens.end(), tagIds.toolCallEndTokenId); - if (itArgs != generatedTokens.end()) { - end = std::distance(generatedTokens.begin(), itArgs); - } else { - break; - } - - std::string toolListStr = tokenizer.decode(std::vector(generatedTokens.begin() + start + 1, generatedTokens.begin() + end), ov::AnyMap{ov::genai::skip_special_tokens(false)}); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool list string: {}", toolListStr); - toolListStr = toolListStr.substr(TOOL_LIST_START_INDICATOR.length(), toolListStr.length() - TOOL_LIST_START_INDICATOR.length() - TOOL_LIST_END_INDICATOR.length()); - - while (!toolListStr.empty()) { - size_t toolEndPos = findInStringRespectingSpecialChars(toolListStr, TOOL_ARGS_END_INDICATOR, 0); - std::string singleTool; - if (toolEndPos != std::string::npos) { - singleTool = toolListStr.substr(0, toolEndPos + TOOL_ARGS_END_INDICATOR.length()); - if (toolEndPos + TOOL_ARGS_END_INDICATOR.length() < toolListStr.length()) { - toolListStr = toolListStr.substr(toolEndPos + TOOL_ARGS_END_INDICATOR.length() + TOOL_SEPARATOR_STR.length()); - } else { - toolListStr.clear(); - } - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed single tool string {}", singleTool); - } else { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more tool calls found in tool list string: {}", toolListStr); - break; - } - - if (!singleTool.empty()) { - tools.push_back(singleTool); - } - } - pos = end; - toolCallPositions.emplace_back(start, end); - } - - for (const std::string& tool : tools) { - ToolCall toolCall; - auto wasToolCallParsed = parseSingleToolCall(tool, toolCall); - if (wasToolCallParsed) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool call - name: {}, args: {}", toolCall.name, toolCall.arguments); - parsedOutput.toolCalls.push_back(toolCall); - } else { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse tool call from string: {}", tool); - } - } - - std::vector contentWithoutToolCalls = generatedTokens; - for (auto it = toolCallPositions.rbegin(); it != toolCallPositions.rend(); ++it) { - contentWithoutToolCalls.erase(contentWithoutToolCalls.begin() + it->first, contentWithoutToolCalls.begin() + it->second + 1); - } - if (tagIds.reasoningEndTokenId.has_value() && tagIds.reasoningStartTokenId.has_value()) { - auto reasoningEndIt = std::find(contentWithoutToolCalls.begin(), contentWithoutToolCalls.end(), tagIds.reasoningEndTokenId.value()); - if (reasoningEndIt != contentWithoutToolCalls.end()) { - contentWithoutToolCalls.erase(contentWithoutToolCalls.begin(), reasoningEndIt + 1); - } else { - auto reasoningStartIt = std::find(contentWithoutToolCalls.begin(), contentWithoutToolCalls.end(), tagIds.reasoningStartTokenId.value()); - if (reasoningStartIt != contentWithoutToolCalls.end()) { - contentWithoutToolCalls.erase(reasoningStartIt, contentWithoutToolCalls.end()); - } - } - } - - parsedOutput.content = tokenizer.decode(contentWithoutToolCalls, ov::AnyMap{ov::genai::skip_special_tokens(true)}); -} -} // namespace ovms diff --git a/src/llm/io_processing/lfm2/lfm2_utils.hpp b/src/llm/io_processing/lfm2/lfm2_utils.hpp deleted file mode 100644 index e6bc49761d..0000000000 --- a/src/llm/io_processing/lfm2/lfm2_utils.hpp +++ /dev/null @@ -1,69 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include "src/llm/io_processing/base_output_parser.hpp" - -namespace ovms { - -extern const std::string TOOL_LIST_START_INDICATOR; -extern const std::string TOOL_LIST_END_INDICATOR; -extern const std::string TOOL_ARGS_START_INDICATOR; -extern const std::string TOOL_ARGS_END_INDICATOR; -extern const std::string TOOL_SEPARATOR_STR; -extern const std::string EOS_TOKEN_STR; -extern const int TOOL_CALL_INDEX_START; - -struct Argument { - std::string name; - std::string value; -}; - -enum class State { - Content, - ToolCallStarted, - ToolCallParameters, - ToolCallEnded, - AfterToolCall -}; - -struct TagIds { - std::string toolCallStartTag; - std::string toolCallEndTag; - int64_t toolCallStartTokenId; - int64_t toolCallEndTokenId; - std::optional reasoningStartTokenId = std::nullopt; - std::optional reasoningEndTokenId = std::nullopt; -}; - -std::string parseArrayParameter(std::string argumentStr); -std::string parseObjectParameter(std::string argumentStr); -std::string normalizeArgStr(const std::string& arg); -void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer); -Argument parseSingleArgument(const std::string& argumentStr); -std::vector parseArguments(const std::string& argumentsStr); -bool parseInContentState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const TagIds& tagIds); -bool parseInToolCallState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState); -bool parseInToolCallParametersState(const std::string& streamingContent, ToolCall& toolCall, size_t& streamingPosition, State& currentState); -bool parseInToolCallEndedState(const std::string& streamingContent, size_t& streamingPosition, State& currentState, const std::string& toolCallEndTag); -rapidjson::Document wrapDeltaContent(const std::string& content); -rapidjson::Document wrapDeltaArgs(const std::string& argsStr, int toolCallIndex); -void cutEOSFromContent(std::string& content); -bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall); -void parseUnaryResponse(ParsedOutput& parsedOutput, const std::vector& generatedTokens, ov::genai::Tokenizer& tokenizer, const TagIds& tagIds); - -} // namespace ovms diff --git a/src/llm/io_processing/llama3/tool_parser.cpp b/src/llm/io_processing/llama3/tool_parser.cpp index 845a6cca7b..4fb43581d2 100644 --- a/src/llm/io_processing/llama3/tool_parser.cpp +++ b/src/llm/io_processing/llama3/tool_parser.cpp @@ -27,79 +27,6 @@ #include "src/stringutils.hpp" namespace ovms { -void Llama3ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // TODO: check if we can rely on decoded <|python_tag|> token to be present in the content, so we can drop multiple detokenizations and copies - // and just extract substrings from the content and modify content in-place - - // We search for botTokenId in the generatedTokens to find tool calls start or check if the content starts with "{" (llama3 sometimes does not generate botTokenId) - auto toolCallsStartPosition = generatedTokens.begin(); - toolCallsStartPosition = generatedTokens.end(); - // Find botTokenId in generated_ids - auto botTokenIt = std::find(generatedTokens.begin(), generatedTokens.end(), botTokenId); - - if (botTokenIt != generatedTokens.end()) { - // Decode the content before botTokenId - std::vector contentTokens(generatedTokens.begin(), botTokenIt); - parsedOutput.content = tokenizer.decode(contentTokens); - // Tokens after botTokenId will be treated as tool calls - toolCallsStartPosition = botTokenIt + 1; - } else { - // If botTokenId is not found, check if model output starts with "{" and if so, assume it's a tool call" - if (!parsedOutput.content.empty() && parsedOutput.content[0] == '{') { - // If model output starts with "{", treat it as a tool call - toolCallsStartPosition = generatedTokens.begin(); - parsedOutput.content.clear(); - } - } - - if (toolCallsStartPosition != generatedTokens.end()) { - std::vector toolCallsTokens(toolCallsStartPosition, generatedTokens.end()); - std::string toolsResponse = tokenizer.decode(toolCallsTokens); - - std::vector tools; - size_t start = 0; - size_t end = 0; - while ((end = toolsResponse.find(separator, start)) != std::string::npos) { - std::string tool = toolsResponse.substr(start, end - start); - if (!tool.empty()) { - tools.push_back(tool); - } - start = end + separator.length(); - } - std::string lastTool = toolsResponse.substr(start); - if (!lastTool.empty()) { - tools.push_back(lastTool); - } - - for (const std::string& tool : tools) { - ToolCall toolCall; - rapidjson::Document toolDoc; - toolDoc.Parse(tool.c_str()); - if (toolDoc.HasParseError()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to parse tool call as JSON"); - continue; - } - if (toolDoc.HasMember("name") && toolDoc["name"].IsString()) { - toolCall.name = toolDoc["name"].GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid name field"); - continue; - } - - if (toolDoc.HasMember("parameters") && toolDoc["parameters"].IsObject()) { - rapidjson::StringBuffer sb; - rapidjson::Writer toolWriter(sb); - toolDoc["parameters"].Accept(toolWriter); - toolCall.arguments = sb.GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid parameters object"); - continue; - } - toolCall.id = generateRandomId(); // Generate a random ID for the tool call - parsedOutput.toolCalls.push_back(toolCall); - } - } -} void Llama3ToolParser::startNextToolCall() { lastJson.Clear(); @@ -121,14 +48,18 @@ static inline void changeParametersToArguments(rapidjson::Document& json) { } } -std::optional Llama3ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { - if (chunk.empty()) { +std::optional Llama3ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { + const bool hasPendingState = + !argumentsDelayWindow[0].empty() || + !argumentsDelayWindow[1].empty() || + jsonHasArgumentsOrParameters(lastJson); + if (chunk.empty() && !hasPendingState) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Llama3ToolParser"); return std::nullopt; } - // <|python_tag|> appears - if (chunk.find(parsingStartTag) != std::string::npos) { + // <|python_tag|> boundary text (synthesised by OutputParser on token-ID detection) + if (chunk.find(parsingConfig.startTags[0]) != std::string::npos) { this->startNextToolCall(); return std::nullopt; // ignoring the special tag } @@ -176,11 +107,19 @@ std::optional Llama3ToolParser::parseChunk(const std::strin // We need to place it right before last closing brace if (finishReason != ov::genai::GenerationFinishReason::NONE) { isCurrentToolCallParsingFinished = true; - size_t lastClosingBrace = modifiedChunk.find_last_of('}'); - if (lastClosingBrace != std::string::npos) { - modifiedChunk.insert(lastClosingBrace, "\""); + if (modifiedChunk.empty()) { + // Empty STOP flush from streamer: finalize the delayed chunk in-place. + size_t lastClosingBrace = argumentsDelayWindow[0].find_last_of('}'); + if (lastClosingBrace != std::string::npos) { + argumentsDelayWindow[0].insert(lastClosingBrace, "\""); + } + } else { + size_t lastClosingBrace = modifiedChunk.find_last_of('}'); + if (lastClosingBrace != std::string::npos) { + modifiedChunk.insert(lastClosingBrace, "\""); + } + argumentsDelayWindow[0] += modifiedChunk; } - argumentsDelayWindow[0] += modifiedChunk; // If this is end of one of the tool calls "in the middle" (; has been found), we need to manually add closing quote " // We need to place it right before last closing brace } else if (modifiedChunk.find(separator) != std::string::npos) { @@ -210,7 +149,6 @@ std::optional Llama3ToolParser::parseChunk(const std::strin throw std::runtime_error("Generated tool call structure is not valid"); // re-throw } - rapidjson::Document doc; // Case 1: 'parameters'/'arguments' has just appeared in the current chunk. If so, we return first delta. if (jsonHasArgumentsOrParameters(newJson) && !jsonHasArgumentsOrParameters(lastJson)) { std::string functionName; @@ -224,10 +162,9 @@ std::optional Llama3ToolParser::parseChunk(const std::strin SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call name has not been generated and parameters already started"); throw std::runtime_error("Tool call name is missing in generated output"); } - // Wrap first delta in {"tool_calls":[{"id":,"type":"function","index":,"function":{"name": }}]} - doc = wrapFirstDelta(functionName, toolCallIndex); + const int currentToolCallIndex = toolCallIndex; lastJson.CopyFrom(newJson, lastJson.GetAllocator()); - return doc; + return ToolCallDelta{currentToolCallIndex, generateRandomId(), functionName, ""}; // Case 2: 'parameters' already exists in the last JSON, we compute delta and return it. } else if (lastJson.HasMember("arguments") || lastJson.HasMember("parameters")) { changeParametersToArguments(newJson); @@ -242,12 +179,14 @@ std::optional Llama3ToolParser::parseChunk(const std::strin return std::nullopt; } } - // Wrap delta in {"tool_calls":[{"index":,"function":}]} - doc = wrapDelta(delta, toolCallIndex); - if (isCurrentToolCallParsingFinished) { + // Wrap delta in {"tool_calls":[{"index":,"function":{"arguments":"..."}}]} + std::string argsStr; + if (delta.HasMember("arguments") && delta["arguments"].IsString()) + argsStr = delta["arguments"].GetString(); + const int currentToolCallIndex = toolCallIndex; + if (isCurrentToolCallParsingFinished) this->startNextToolCall(); - } - return doc; + return ToolCallDelta{currentToolCallIndex, std::nullopt, std::nullopt, argsStr}; // Case 3: No 'parameters' exists or just appeared, so we keep building up until we have complete function name } else { lastJson.CopyFrom(newJson, lastJson.GetAllocator()); diff --git a/src/llm/io_processing/llama3/tool_parser.hpp b/src/llm/io_processing/llama3/tool_parser.hpp index 40d235d411..bfcf902ad5 100644 --- a/src/llm/io_processing/llama3/tool_parser.hpp +++ b/src/llm/io_processing/llama3/tool_parser.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -28,13 +29,6 @@ namespace ovms { class Llama3ToolParser : public BaseOutputParser { protected: - const std::string parsingStartTag = "<|python_tag|>"; - // Tools calls are expected to be the last part of the content and there is no unique separator between tools, so we do not specify an end tag. - const std::string parsingEndTag = ""; - - // Id of the <|python_tag|> which is a special token used to indicate the start of a tool calls - int64_t botTokenId = 128010; - // ";" is used as a separator between tool calls in the response std::string separator = ";"; // Streaming required members @@ -51,22 +45,34 @@ class Llama3ToolParser : public BaseOutputParser { public: Llama3ToolParser() = delete; - explicit Llama3ToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags = {parsingStartTag}; - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags = {"{"}; - return specialParsingStartTags; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + // <|python_tag|> is a special token. Put it in both startTags (text-based, + // used when the text is passed directly, e.g. in streaming tests) and + // tokenIdStartTags (token-ID-based, used in production where the token + // decodes to empty with skip_special_tokens=true). + cfg.startTags = {"<|python_tag|>"}; + cfg.tokenIdStartTags = {"<|python_tag|>"}; + cfg.preambleStartTags = {"{"}; + return cfg; } - // Tools calls are expected to be the last part of the content, so we do not specify an end tag. - const std::string& getParsingEndTag() const override { - return parsingEndTag; + + explicit Llama3ToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + lastJson.SetNull(); + jsonBuilder.clear(); + toolCallIndex = -1; + argumentsDelayWindow = {{" ", ""}}; + argumentsDelayWindow[0].clear(); + argumentsDelayWindow[1].clear(); + escapeLevel = 0; } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.cpp b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.cpp deleted file mode 100644 index 11c4218e1e..0000000000 --- a/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.cpp +++ /dev/null @@ -1,81 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include - -#include "src/port/rapidjson_document.hpp" - -#include "src/logging.hpp" -#include "minicpm5_reasoning_parser.hpp" -#include "src/llm/io_processing/utils.hpp" - -namespace ovms { -void Minicpm5ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - auto startReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningStartTokenId); - auto endReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId); - - if ((startReasoningIt == generatedTokens.end() && endReasoningIt == generatedTokens.end())) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ReasoningParser: Reasoning start or end token not found in the generated tokens. Start token found: {}, End token found: {}, Start position: {}, End position: {}", - startReasoningIt != generatedTokens.end(), endReasoningIt != generatedTokens.end(), std::distance(generatedTokens.begin(), startReasoningIt), std::distance(generatedTokens.begin(), endReasoningIt)); - return; - } - - auto startPos = 0; - if (startReasoningIt != generatedTokens.end()) { - startPos = std::distance(generatedTokens.begin(), startReasoningIt) + 1; - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ReasoningParser: Reasoning start token not found in the generated tokens. Start position: {}", startPos); - } - auto endPos = std::distance(generatedTokens.begin(), endReasoningIt); - - std::string reasoningContent = tokenizer.decode(std::vector(startPos + generatedTokens.begin(), endPos + generatedTokens.begin()), ov::genai::skip_special_tokens(true)); - - parsedOutput.reasoning = reasoningContent; - - if (endReasoningIt != generatedTokens.end()) { - endPos += 1; - } - - std::string contentWithoutReasoning = tokenizer.decode(std::vector(endPos + generatedTokens.begin(), generatedTokens.end()), ov::genai::skip_special_tokens(true)); - parsedOutput.content = contentWithoutReasoning; -} - -std::optional Minicpm5ReasoningParser::parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) { - if (tokens.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty tokens for Minicpm5ReasoningParser"); - return std::nullopt; - } - - if (std::find(tokens.begin(), tokens.end(), reasoningStartTokenId) != tokens.end() || - std::find(tokens.begin(), tokens.end(), reasoningEndTokenId) != tokens.end()) { - return std::nullopt; - } else { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("reasoning_content"); - writer.String(chunk.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - return doc; - } -} -} // namespace ovms diff --git a/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp index fe194638f8..d7b1ccfb45 100644 --- a/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp +++ b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp @@ -14,40 +14,33 @@ // limitations under the License. //***************************************************************************** #pragma once -#include "src/llm/io_processing/base_output_parser.hpp" -#include +#include "src/llm/io_processing/qwen3/reasoning_parser.hpp" +#include #include +#include +#include namespace ovms { -class Minicpm5ReasoningParser : public BaseOutputParser { -public: - static inline const std::string reasoningStartTag = ""; - static inline const std::string reasoningEndTag = ""; - - static constexpr int64_t reasoningStartTokenId = 8; - static constexpr int64_t reasoningEndTokenId = 9; - +// MiniCPM5 reasoning uses the same /<\think> grammar as Qwen3 but both delimiters +// are registered special tokens, so tokenIdStartTags is set and needsSpecialTokens/ +// defaultDecodingWithSpecialTokens are both true. +class Minicpm5ReasoningParser : public Qwen3ReasoningParser { public: Minicpm5ReasoningParser() = delete; - explicit Minicpm5ReasoningParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{this->reasoningStartTag}; - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - return reasoningEndTag; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {""}; + cfg.tokenIdStartTags = {""}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + explicit Minicpm5ReasoningParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + Qwen3ReasoningParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} }; } // namespace ovms diff --git a/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp index 08e19c27a3..36258dd6ec 100644 --- a/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp +++ b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp @@ -291,18 +291,11 @@ std::optional Minicpm5ToolParserImpl::getCurrentFunctionName() cons // ---- Minicpm5ToolParser ---- -void Minicpm5ToolParser::lazyFillInitToolParametersTypesMap() { - if (this->filledParametersTypesMap) - return; - SPDLOG_DEBUG("Minicpm5ToolParser: filling tools parameters types map"); - this->toolsParametersTypes = createToolsParametersTypesMap(this->toolSchemas); - this->filledParametersTypesMap = true; - SPDLOG_DEBUG("Minicpm5ToolParser: created with {} tools", this->toolsParametersTypes.size()); -} - Minicpm5ToolParser::Minicpm5ToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : - BaseOutputParser(tokenizer), + BaseOutputParser(tokenizer, + defaultParsingConfig()), toolSchemas(toolSchemas), + toolsParametersTypes(createToolsParametersTypesMap(toolSchemas)), streamParser(this->toolsParametersTypes) {} const std::vector Minicpm5ToolParser::removeReasoningTokens(const std::vector& generatedTokens) { @@ -328,25 +321,7 @@ const std::vector Minicpm5ToolParser::removeReasoningTokens(const std:: return tokensWithoutReasoning; } -void Minicpm5ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - auto tokensWithoutReasoning = this->removeReasoningTokens(generatedTokens); - std::string contentWithSpecialTokens = this->tokenizer.decode(tokensWithoutReasoning, ov::genai::skip_special_tokens(false)); - this->lazyFillInitToolParametersTypesMap(); - auto toolCallsOpt = this->streamParser.parseChunk(contentWithSpecialTokens); - if (toolCallsOpt.has_value()) { - parsedOutput.toolCalls = std::move(toolCallsOpt.value()); - SPDLOG_DEBUG("Minicpm5ToolParser: parse done, removing tool calls from content"); - auto status = this->streamParser.removeToolCallsFromContentIfNeeded(contentWithSpecialTokens); - if (!status.ok()) { - SPDLOG_DEBUG("Minicpm5ToolParser: failed to remove tool calls from content: {}", status.string()); - } - parsedOutput.content = std::move(contentWithSpecialTokens); - return; - } - SPDLOG_DEBUG("Minicpm5ToolParser: parse done, no tool calls found"); -} - -std::optional Minicpm5ToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { +std::optional Minicpm5ToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { if (toolCalls.size() != 1) { SPDLOG_ERROR("Minicpm5ToolParser: for streaming expected one tool call, got: {}", toolCalls.size()); throw std::runtime_error("Minicpm5ToolParser: for streaming expected one tool call"); @@ -362,66 +337,32 @@ std::optional Minicpm5ToolParser::sendFullDelta(const ToolC return wrapCombinedDelta(toolCall); } this->returnedCompleteDeltas.insert(this->toolCallIndex); - rapidjson::Document argumentsWrapper; - argumentsWrapper.SetObject(); - rapidjson::Document::AllocatorType& allocator = argumentsWrapper.GetAllocator(); - rapidjson::Value toolCallsString(rapidjson::kStringType); - toolCallsString.SetString(toolCall.arguments.c_str(), allocator); SPDLOG_TRACE("Minicpm5ToolParser: tool call arguments string: {}", toolCall.arguments); - argumentsWrapper.AddMember("arguments", toolCallsString, allocator); - auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); - SPDLOG_DEBUG("Minicpm5ToolParser: full delta: {}", documentToString(currentDelta)); - return currentDelta; + SPDLOG_DEBUG("Minicpm5ToolParser: full delta: index={} arguments={}", this->toolCallIndex, toolCall.arguments); + return ToolCallDelta{this->toolCallIndex, std::nullopt, std::nullopt, toolCall.arguments}; } -rapidjson::Document Minicpm5ToolParser::wrapCombinedDelta(const ToolCall& toolCall) { - rapidjson::Document wrappedDelta; - wrappedDelta.SetObject(); - rapidjson::Document::AllocatorType& allocator = wrappedDelta.GetAllocator(); - - rapidjson::Value toolCalls(rapidjson::kArrayType); - rapidjson::Value toolCallObj(rapidjson::kObjectType); - rapidjson::Value idValue(generateRandomId().c_str(), allocator); - toolCallObj.AddMember("id", idValue, allocator); - toolCallObj.AddMember("type", "function", allocator); - toolCallObj.AddMember("index", this->toolCallIndex, allocator); - - rapidjson::Value functionObj(rapidjson::kObjectType); - rapidjson::Value nameValue(toolCall.name.c_str(), allocator); - functionObj.AddMember("name", nameValue, allocator); - - rapidjson::Value argumentsValue(rapidjson::kStringType); - argumentsValue.SetString(toolCall.arguments.c_str(), allocator); - functionObj.AddMember("arguments", argumentsValue, allocator); - toolCallObj.AddMember("function", functionObj, allocator); - - toolCalls.PushBack(toolCallObj, allocator); - rapidjson::Value deltaWrapper(rapidjson::kObjectType); - deltaWrapper.AddMember("tool_calls", toolCalls, allocator); - wrappedDelta.AddMember("delta", deltaWrapper, allocator); - SPDLOG_DEBUG("Minicpm5ToolParser: combined delta: {}", documentToString(wrappedDelta)); - return wrappedDelta; +ToolCallDelta Minicpm5ToolParser::wrapCombinedDelta(const ToolCall& toolCall) { + SPDLOG_DEBUG("Minicpm5ToolParser: combined delta: index={} name={} args={}", this->toolCallIndex, toolCall.name, toolCall.arguments); + return ToolCallDelta{this->toolCallIndex, generateRandomId(), toolCall.name, toolCall.arguments}; } -std::optional Minicpm5ToolParser::sendFirstDeltaIfNeeded(const std::string& toolCallName) { +std::optional Minicpm5ToolParser::sendFirstDeltaIfNeeded(const std::string& toolCallName) { if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { SPDLOG_TRACE("Minicpm5ToolParser: skipping first delta, already sent for current function"); return std::nullopt; } int toolCallId = ++this->toolCallIndex; - rapidjson::Document doc = wrapFirstDelta(toolCallName, toolCallId); - this->currentJson.CopyFrom(doc, this->currentJson.GetAllocator()); this->returnedFirstDeltas.insert(toolCallId); - SPDLOG_DEBUG("Minicpm5ToolParser: first delta: {}", documentToString(doc)); - return doc; + SPDLOG_DEBUG("Minicpm5ToolParser: first delta: name={} index={}", toolCallName, toolCallId); + return ToolCallDelta{toolCallId, generateRandomId(), toolCallName, ""}; } -std::optional Minicpm5ToolParser::parseChunk( +std::optional Minicpm5ToolParser::parseChunk( const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { SPDLOG_DEBUG("Minicpm5ToolParser: chunk: '{}'", newChunk); - this->lazyFillInitToolParametersTypesMap(); if (newChunk.empty()) return std::nullopt; auto toolCallsOpt = this->streamParser.parseChunk(newChunk); diff --git a/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp index 4e1cf972c0..bbc8e56750 100644 --- a/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp +++ b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp @@ -85,6 +85,15 @@ struct Minicpm5ToolParserImpl { Status removeToolCallsFromContentIfNeeded(std::string& outContent); + void reset() { + currentState = State::Content; + currentFunction.clear(); + currentParameterName.clear(); + streamContent.clear(); + lastProcessedPosition = 0; + toolCallPositions = ToolCallPositions{}; + } + State getCurrentState() const { return this->currentState; } size_t getLastProcessedPosition() const { return this->lastProcessedPosition; } @@ -136,49 +145,40 @@ class Minicpm5ToolParser : public BaseOutputParser { private: const ToolsSchemas_t& toolSchemas; ToolsParameterTypeMap_t toolsParametersTypes; - bool filledParametersTypesMap{false}; Minicpm5ToolParserImpl streamParser; int toolCallIndex{-1}; ToolCalls_t currentToolCalls; - rapidjson::Document currentJson; std::set returnedFirstDeltas; std::set returnedCompleteDeltas; public: Minicpm5ToolParser() = delete; - explicit Minicpm5ToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas); - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - - const std::vector& getParsingStartTags() const override { - static const std::vector startTags = {FUNCTION_START_TAG}; - return startTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags = {}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - static const std::string EMPTY_STRING = ""; - return EMPTY_STRING; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {FUNCTION_START_TAG}; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - bool requiresStreamingWithSpecialTokens() const override { - return true; - } + explicit Minicpm5ToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas); - const std::vector& getSpecialTagsToErase() const override { - static const std::vector tagsToErase = {SOS_TOKEN_STR, EOS_TOKEN_STR}; - return tagsToErase; + void resetState() override { + streamParser.reset(); + toolCallIndex = -1; + currentToolCalls.clear(); + returnedFirstDeltas.clear(); + returnedCompleteDeltas.clear(); } + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + private: const std::vector removeReasoningTokens(const std::vector& generatedTokens); - std::optional sendFirstDeltaIfNeeded(const std::string& currentFunctionName); - std::optional sendFullDelta(const ToolCalls_t& toolCalls); - rapidjson::Document wrapCombinedDelta(const ToolCall& toolCall); - void lazyFillInitToolParametersTypesMap(); + std::optional sendFirstDeltaIfNeeded(const std::string& currentFunctionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + ToolCallDelta wrapCombinedDelta(const ToolCall& toolCall); }; } // namespace ovms diff --git a/src/llm/io_processing/mistral/tool_parser.cpp b/src/llm/io_processing/mistral/tool_parser.cpp index 11ba39979c..981c34234a 100644 --- a/src/llm/io_processing/mistral/tool_parser.cpp +++ b/src/llm/io_processing/mistral/tool_parser.cpp @@ -28,59 +28,6 @@ namespace ovms { -void MistralToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - std::vector tools; - - if (parsedOutput.content.empty() || generatedTokens.size() <= 0) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "No content to parse for tool calls"); - return; - } - - // Parser will consume entire model output only if the first generated token is the beginning of tools token. - if (generatedTokens[0] != this->botTokenId) { - if (parsedOutput.content.size() >= 2 && parsedOutput.content[0] == '[' && parsedOutput.content[1] == '{') { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Model output starts with '[{' but begin of tools token is missing. Proceeding with parsing."); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Begin of tools token or '[{' has not been found in the model output. Exiting parser."); - return; - } - } - - rapidjson::Document toolsDoc; - toolsDoc.Parse(parsedOutput.content.c_str()); - - if (!toolsDoc.HasParseError() && toolsDoc.IsArray()) { - for (auto& toolVal : toolsDoc.GetArray()) { - if (!toolVal.IsObject()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call is not a valid JSON object"); - continue; - } - ToolCall toolCall; - if (toolVal.HasMember("name") && toolVal["name"].IsString()) { - toolCall.name = toolVal["name"].GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid name field"); - continue; - } - - if (toolVal.HasMember("arguments") && toolVal["arguments"].IsObject()) { - rapidjson::StringBuffer sb; - rapidjson::Writer toolWriter(sb); - toolVal["arguments"].Accept(toolWriter); - toolCall.arguments = sb.GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid parameters object"); - continue; - } - toolCall.id = generateRandomId(); // Generate a random ID for the tool call - parsedOutput.toolCalls.push_back(toolCall); - } - parsedOutput.content.clear(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to parse functools content or extract tools array"); - } -} - void MistralToolParser::movePostColonContentToUnprocessedBuffer(std::string& chunk) { size_t colonPos = chunk.find(':'); if (colonPos != std::string::npos) { @@ -162,7 +109,7 @@ void MistralToolParser::clearState() { openBracesCount = 1; // Reset to 1 as we count the tool call opening brace } -std::optional MistralToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional MistralToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { /* Mistral with vLLM template produces tool calls in the format (beginning [TOOL_CALL] is skipped by the mode or just not visible during streaming): [{"name": [function name], "arguments": [function arguments as JSON]}, ...] @@ -180,7 +127,13 @@ std::optional MistralToolParser::parseChunk(const std::stri We address this by escaping double quotes and adding opening quote at the beginning of arguments and closing quote at the end of arguments. */ SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "MistralToolParser::parseChunk called with chunk: '{}', finishReason: {}", chunk, static_cast(finishReason)); - if (chunk.empty()) { + const bool hasPendingState = + !unprocessedBuffer.empty() || + (internalState == PROCESSING_TOOL_CALL && lastJson.HasMember("arguments")); + + // Empty chunks are normally ignorable, except finalization calls when we still + // have buffered/parser state to flush (e.g. empty STOP chunk from streamer). + if (chunk.empty() && !hasPendingState) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for MistralToolParser"); return std::nullopt; } @@ -199,11 +152,20 @@ std::optional MistralToolParser::parseChunk(const std::stri // Phase 1: Control the internal state and apply changes to the chunk if needed if (internalState == AWAITING_START_TAG) { - // We did not see "[{" yet, so we look for it in the current chunk - if (modifiedChunk.find(streamingParsingStartTag) != std::string::npos) { - // We found "[{", so we switch to the the state where we are waiting for the opening bracket of the array + // We did not see a start marker yet; accept either visible [TOOL_CALLS] + // token or direct JSON array/object prefix "[{". + const std::string visibleStartTag = "[TOOL_CALLS]"; + if (modifiedChunk.find(visibleStartTag) != std::string::npos || modifiedChunk.find(streamingParsingStartTag) != std::string::npos) { + // Start marker found, switch to the state waiting for array opening bracket. internalState = AWAITING_TOOL_CALLS_OPENING_BRACKET; - // We have more content in the chunk after "[{", so we process the rest of the chunk in the next state + + // If the visible [TOOL_CALLS] token is present, drop it before further processing. + size_t visibleStartPos = modifiedChunk.find(visibleStartTag); + if (visibleStartPos != std::string::npos) { + modifiedChunk.erase(visibleStartPos, visibleStartTag.length()); + } + + // Continue processing the remaining content in the next state. return parseChunk(modifiedChunk, {}, finishReason); } return std::nullopt; @@ -250,6 +212,7 @@ std::optional MistralToolParser::parseChunk(const std::stri escapeSpecialCharacters(modifiedChunk); // Keep track of opened/closed braces to identify the end of the tool call object. + const size_t openBracesCountBeforeUpdate = openBracesCount; updateOpenBracesCount(modifiedChunk); // When we start collecting arguments, force string type by adding opening quote @@ -260,7 +223,7 @@ std::optional MistralToolParser::parseChunk(const std::stri if (finishReason != ov::genai::GenerationFinishReason::NONE) { handleGenerationFinish(modifiedChunk); - } else if (openBracesCount == 0) { + } else if (openBracesCount == 0 && openBracesCountBeforeUpdate > 0) { // If we balanced the braces, we are at the end of the tool call object handleEndOfToolCall(modifiedChunk); } @@ -280,7 +243,6 @@ std::optional MistralToolParser::parseChunk(const std::stri throw std::runtime_error("Generated tool call structure is not valid"); } - rapidjson::Document doc; // Case 1: 'arguments' has just appeared in the current chunk. If so, we return first delta. if (newJson.HasMember("arguments") && !lastJson.HasMember("arguments")) { std::string functionName; @@ -294,9 +256,8 @@ std::optional MistralToolParser::parseChunk(const std::stri throw std::runtime_error("Tool call name is missing in generated output"); } // Wrap first delta in {"tool_calls":[{"id":,"type":"function","index":,"function":{"name": }}]} - doc = wrapFirstDelta(functionName, toolCallIndex); lastJson.CopyFrom(newJson, lastJson.GetAllocator()); - return doc; + return ToolCallDelta{toolCallIndex, generateRandomId(), functionName, ""}; // Case 2: 'arguments' already exists in the last JSON, we compute delta and return it. } else if (lastJson.HasMember("arguments")) { rapidjson::Document delta = PartialJsonBuilder::computeDelta(lastJson, newJson); @@ -321,9 +282,11 @@ std::optional MistralToolParser::parseChunk(const std::stri } } - // Wrap delta in {"tool_calls":[{"index":,"function":}]} - doc = wrapDelta(delta, toolCallIndex); - return doc; + // Wrap delta in {"tool_calls":[{"index":,"function":{"arguments":"..."}}]} + std::string argsStr; + if (delta.HasMember("arguments") && delta["arguments"].IsString()) + argsStr = delta["arguments"].GetString(); + return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, argsStr}; // Case 3: No 'arguments' exists or just appeared, so we keep building up until we have complete function name } else { lastJson.CopyFrom(newJson, lastJson.GetAllocator()); diff --git a/src/llm/io_processing/mistral/tool_parser.hpp b/src/llm/io_processing/mistral/tool_parser.hpp index 8f1a762f85..4e5d467814 100644 --- a/src/llm/io_processing/mistral/tool_parser.hpp +++ b/src/llm/io_processing/mistral/tool_parser.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -66,23 +67,27 @@ class MistralToolParser : public BaseOutputParser { public: MistralToolParser() = delete; - explicit MistralToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector toolCallStartTags{"[TOOL_CALLS]", streamingParsingStartTag}; - return toolCallStartTags; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.tokenIdStartTags = {"[TOOL_CALLS]"}; + cfg.startTags = {"[TOOL_CALLS]", "[{\""}; // [TOOL_CALLS] for direct text, [{" as fallback + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - // Tools calls are expected to be the last part of the content, so we do not specify an end tag. - const std::string& getParsingEndTag() const override { - static const std::string toolCallEndTag = ""; - return toolCallEndTag; + + explicit MistralToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + internalState = AWAITING_START_TAG; + lastJson.SetNull(); + jsonBuilder.clear(); + toolCallIndex = -1; + argumentsQuotesOpened = false; } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_content_parser.cpp b/src/llm/io_processing/onyx/onyx_content_parser.cpp new file mode 100644 index 0000000000..40552a725c --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_content_parser.cpp @@ -0,0 +1,77 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "src/stringutils.hpp" + +#include "onyx_content_parser.hpp" + +namespace ovms { + +namespace { + +void eraseTags(std::string& content, const std::vector& tags) { + for (const auto& tag : tags) { + size_t pos = 0; + while ((pos = content.find(tag, pos)) != std::string::npos) + content.erase(pos, tag.size()); + } +} + +static const std::vector HOLD_TAGS = { + "<|eom|>", "<|start|>assistant ", "<|message|>", "<|eot|>"}; + +static const std::vector ROUTING_TAGS = {" to=user", "to=user"}; + +} // namespace + +OutputParsingConfig OnyxContentParser::defaultParsingConfig() { + OutputParsingConfig cfg; + // Resume signal: detected in TOOL_CALLS_WAITING_FOR_TOOL to switch back to content. + cfg.startTags = {"to=user<|message|>"}; + cfg.needsSpecialTokens = true; + return cfg; +} + +OnyxContentParser::OnyxContentParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + +std::optional OnyxContentParser::parseChunk( + const std::string& buffer, + const std::vector& /*tokens*/, + ov::genai::GenerationFinishReason /*finishReason*/) { + + bool anyComplete = false; + for (const auto& tag : HOLD_TAGS) { + if (buffer.find(tag) != std::string::npos) { + anyComplete = true; + } else if (stringsOverlap(buffer, tag)) { + return std::nullopt; // partial match — hold until tag is fully assembled + } + } + + std::string content = buffer; + if (anyComplete) + eraseTags(content, HOLD_TAGS); + eraseTags(content, ROUTING_TAGS); + + return ContentDelta{std::move(content)}; +} + +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_content_parser.hpp b/src/llm/io_processing/onyx/onyx_content_parser.hpp new file mode 100644 index 0000000000..1c4a6e75f7 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_content_parser.hpp @@ -0,0 +1,58 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +#include + +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { + +// Handles plain-content turns in the Onyx harmony format. +// +// Strips the routing preamble ("<|eom|><|start|>assistant to=user<|message|>") that +// precedes user-visible content and the terminator ("<|eot|>") that follows it. +// +// Tag handling uses two tiers: +// structural — hold-eligible: "<|eom|>", "<|start|>assistant ", "<|message|>", "<|eot|>" +// A partial match at the END of the buffer causes a hold until the tag +// is fully assembled (or definitively not present). +// routing — immediate-erase: " to=user", "to=user" +// Erased whenever fully present; never trigger a hold, because their +// leading space would cause false FOUND_INCOMPLETE hits on any content +// token that ends with a space. +// +// startTags = {"to=user<|message|>"} — used by OutputParser::TOOL_CALLS_WAITING_FOR_TOOL +// to detect when the model switches from tool calls back to a plain content turn. +class OnyxContentParser : public BaseOutputParser { +public: + OnyxContentParser() = delete; + explicit OnyxContentParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt); + + static OutputParsingConfig defaultParsingConfig(); + + // Returns a content delta document, or nullopt to hold (partial structural-tag match). + // Returns a document with empty content string when the buffer contained only preamble + // (caller suppresses the emit in that case). + std::optional parseChunk(const std::string& buffer, + const std::vector& tokens, + ov::genai::GenerationFinishReason finishReason) override; +}; + +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp index 05a4a5db8a..e9cbb2e08c 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -18,108 +18,45 @@ #include #include -#include "src/port/rapidjson_document.hpp" - #include "src/logging.hpp" #include "onyx_reasoning_parser.hpp" namespace ovms { -void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // Step 1: Extract and remove ALL "to=self<|message|>...<|eom|>" reasoning segments. - for (;;) { - size_t selfPos = parsedOutput.content.find(selfRecipientTag); - if (selfPos == std::string::npos) - break; - size_t messagePos = parsedOutput.content.find(messageTag, selfPos); - if (messagePos == std::string::npos) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Found '{}' without a following '{}', leaving content untouched", selfRecipientTag, messageTag); - break; - } - size_t bodyStart = messagePos + messageTag.length(); - size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); - std::string reasoning = (endPos != std::string::npos) - ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) - : parsedOutput.content.substr(bodyStart); - if (!parsedOutput.reasoning.empty()) - parsedOutput.reasoning += '\n'; - parsedOutput.reasoning += reasoning; - // Erase the segment including the leading space before "to=" if present. - size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; - size_t eraseEnd = (endPos != std::string::npos) ? endPos + continuationEndTag.length() : parsedOutput.content.length(); - parsedOutput.content.erase(segmentStart, eraseEnd - segmentStart); - } - - // Step 2: Remove all "<|start|>assistant" turn boundary markers (with optional trailing space). - static const std::string turnBoundary = "<|start|>assistant"; - for (;;) { - size_t pos = parsedOutput.content.find(turnBoundary); - if (pos == std::string::npos) - break; - size_t eraseLen = turnBoundary.length(); - // Also consume one trailing space if present (before "to="). - if (pos + eraseLen < parsedOutput.content.length() && parsedOutput.content[pos + eraseLen] == ' ') - ++eraseLen; - parsedOutput.content.erase(pos, eraseLen); +std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for OnyxReasoningParser"); + return std::nullopt; } - // Step 3: Strip envelope framing from remaining turns. Each non-self turn has - // " to=<|message|>" before its body. Find each "<|message|>" tag, look - // backwards for the closest "to=" prefix, and erase the envelope (including a - // leading space if present). This preserves content between tool-call turns. - static const std::string toPrefix = "to="; - // The envelope (" to=<|message|>") is never longer than this. - static constexpr size_t maxEnvelopeLen = 128; - for (;;) { - size_t messagePos = parsedOutput.content.find(messageTag); - if (messagePos == std::string::npos) - break; - // Bound the backwards search to avoid matching "to=" in body content. - size_t searchFrom = (messagePos > maxEnvelopeLen) ? messagePos - maxEnvelopeLen : 0; - size_t toPos = parsedOutput.content.rfind(toPrefix, messagePos); - size_t eraseStart; - if (toPos != std::string::npos && toPos >= searchFrom && parsedOutput.content.find(messageTag, toPos) == messagePos) { - // Include the leading space before "to=" if present. - eraseStart = (toPos > 0 && parsedOutput.content[toPos - 1] == ' ') ? toPos - 1 : toPos; - } else { - // No "to=" found within the envelope window; erase just the tag itself. - eraseStart = messagePos; + // Buffer content until the <|message|> separator has been fully consumed. + // The tag arrives as individual tokens so we must accumulate to detect the full string. + if (!headerConsumed) { + headerBuffer += chunk; + size_t msgPos = headerBuffer.find(messageTag); + if (msgPos == std::string::npos) { + return std::nullopt; } - parsedOutput.content.erase(eraseStart, messagePos + messageTag.length() - eraseStart); - } - - // Step 4: Remove all remaining terminators. - for (const auto& term : {continuationEndTag, turnFinalEndTag}) { - for (;;) { - size_t pos = parsedOutput.content.find(term); - if (pos == std::string::npos) - break; - parsedOutput.content.erase(pos, term.length()); + headerConsumed = true; + std::string afterMsg = headerBuffer.substr(msgPos + messageTag.size()); + headerBuffer.clear(); + if (afterMsg.empty()) { + return std::nullopt; } + // Fall through with the content that follows the separator. + return parseChunk(afterMsg, {}, ov::genai::GenerationFinishReason::NONE); } -} -std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { - if (chunk.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for OnyxReasoningParser"); - return std::nullopt; + // Strip the end tag if it arrives bundled with the last reasoning text. + std::string text = chunk; + const size_t endTagPos = text.rfind(continuationEndTag); + if (endTagPos != std::string::npos) { + text = text.substr(0, endTagPos); } - if (chunk.find(selfRecipientTag) != std::string::npos || - chunk.find(messageTag) != std::string::npos || - chunk.find(continuationEndTag) != std::string::npos) { + if (text.empty()) { return std::nullopt; } - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("reasoning_content"); - writer.String(chunk.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - return doc; + + return ReasoningDelta{text}; } } // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp index e7b7d415a2..4f36166ee8 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -28,7 +29,6 @@ namespace ovms { class OnyxReasoningParser : public BaseOutputParser { -protected: // Marks a private chain-of-thought turn (recipient="self"). const std::string selfRecipientTag = "to=self"; // Separates the routing prefix from the turn's body. @@ -40,24 +40,31 @@ class OnyxReasoningParser : public BaseOutputParser { public: OnyxReasoningParser() = delete; - explicit OnyxReasoningParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{selfRecipientTag}; - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - return continuationEndTag; + + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"to=self"}; + cfg.endTag = "<|eom|>"; + cfg.needsSpecialTokens = true; + cfg.defaultDecodingWithSpecialTokens = true; + return cfg; } - bool requiresStreamingWithSpecialTokens() const override { - return true; + + explicit OnyxReasoningParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + headerConsumed = false; + headerBuffer.clear(); } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + +private: + // Accumulates content until <|message|> is fully consumed at the start of each turn. + bool headerConsumed = false; + std::string headerBuffer; }; } // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 7e6bfed7fa..11adfb90ab 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -245,72 +245,40 @@ Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outCo return StatusCode::OK; } -OnyxToolParser::OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : - BaseOutputParser(tokenizer), +OnyxToolParser::OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas, + std::optional configOverride) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()), toolSchemas(toolSchemas), + toolsParametersTypes(createToolsParametersTypesMap(toolSchemas)), streamParser(this->toolsParametersTypes) { + buildStartTags(); } -void OnyxToolParser::lazyFillParsingStartTags() const { - // toolSchemas is a reference that is empty when this object is constructed and only gets - // populated by the caller afterwards, once the current request's tools are known (and may - // hold a different tool set on every request if this parser instance is reused). Rebuild - // "to=" start tags from whatever toolSchemas currently holds on every call instead of - // hardcoding tool names or building the list once too early in the constructor. The schema - // map is small, so recomputing this each time is cheap. - parsingStartTags.clear(); - parsingStartTags.push_back(TOOL_START_TAG); +void OnyxToolParser::buildStartTags() { + parsingConfig.startTags.clear(); + parsingConfig.startTags.push_back(TOOL_START_TAG); for (const auto& [name, _] : toolSchemas) { if (name == "user" || name == "self") { - SPDLOG_DEBUG("Skipping tool name: {} for parsingStartTags", name); + SPDLOG_DEBUG("Skipping tool name: {} for start tags", name); continue; } - parsingStartTags.push_back("to=" + name); + parsingConfig.startTags.push_back("to=" + name); } } -void OnyxToolParser::lazyFillInitToolParametersTypesMap() { - if (this->filledParametersTypesMap) { - return; - } - this->toolsParametersTypes = createToolsParametersTypesMap(this->toolSchemas); - this->filledParametersTypesMap = true; - SPDLOG_DEBUG("OnyxToolParser created with {} tools", this->toolsParametersTypes.size()); -} - -void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // Unary is the single-shot edge case of streaming: drive the same streamParser with the - // whole content as one chunk (mirrors Qwen3CoderToolParser::parse()). - this->lazyFillInitToolParametersTypesMap(); - auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); - if (!toolCallsOpt.has_value()) { - SPDLOG_DEBUG("Parsing ended, no tool calls found"); - return; - } - parsedOutput.toolCalls = std::move(toolCallsOpt.value()); - for (const auto& toolCall : parsedOutput.toolCalls) { - SPDLOG_DEBUG("Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); - } - auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); - if (!status.ok()) { - SPDLOG_DEBUG("Failed to remove tool calls from content: {}", status.string()); - } -} - -std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { +std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { - // already sent the first delta for the function currently being read SPDLOG_TRACE("Skipping first delta, already sent for current function, returnedFirstDeltas.size(): {} returnedCompleteDeltas.size(): {}", returnedFirstDeltas.size(), returnedCompleteDeltas.size()); return std::nullopt; } int currentToolCallIndex = ++this->toolCallIndex; - rapidjson::Document doc = wrapFirstDelta(functionName, currentToolCallIndex); this->returnedFirstDeltas.insert(currentToolCallIndex); - SPDLOG_DEBUG("First delta doc: {}", documentToString(doc)); - return doc; + SPDLOG_DEBUG("First delta: name={} index={}", functionName, currentToolCallIndex); + return ToolCallDelta{currentToolCallIndex, generateRandomId(), functionName, ""}; } -std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { +std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { // ASSUMPTION (mirrors Qwen3CoderToolParser): in streaming we only ever complete one tool // call per parseChunk() -- there is no way to send multiple tool calls in one delta. if (toolCalls.size() != 1) { @@ -319,21 +287,15 @@ std::optional OnyxToolParser::sendFullDelta(const ToolCalls } const auto& toolCall = toolCalls[0]; this->returnedCompleteDeltas.insert(this->toolCallIndex); - rapidjson::Document argumentsWrapper; - argumentsWrapper.SetObject(); - rapidjson::Value argumentsValue(toolCall.arguments.c_str(), static_cast(toolCall.arguments.size()), argumentsWrapper.GetAllocator()); SPDLOG_TRACE("Tool call arguments string: {}", toolCall.arguments); - argumentsWrapper.AddMember("arguments", argumentsValue, argumentsWrapper.GetAllocator()); - auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); - SPDLOG_DEBUG("Full delta doc: {}", documentToString(currentDelta)); - return currentDelta; + SPDLOG_DEBUG("Full delta: index={} arguments={}", this->toolCallIndex, toolCall.arguments); + return ToolCallDelta{this->toolCallIndex, std::nullopt, std::nullopt, toolCall.arguments}; } -std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { // streamParser returns assembled toolCalls once a call closes (""); // until then, if the function name is already known, send its first delta once. SPDLOG_DEBUG("Chunk: '{}', finishReason: {}", newChunk, static_cast(finishReason)); - this->lazyFillInitToolParametersTypesMap(); if (newChunk.empty()) { return std::nullopt; } diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index e8cf635d73..34b8c1c6fd 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -25,8 +25,6 @@ #include -#include "src/port/rapidjson_document.hpp" - #include "src/llm/io_processing/base_output_parser.hpp" #include "src/llm/apis/tool_schema_wrapper.hpp" #include "src/logging.hpp" @@ -94,6 +92,14 @@ struct OnyxToolParserImpl { std::optional parseChunk(const std::string& chunk); std::optional getCurrentFunctionName() const; Status removeToolCallsFromContentIfNeeded(std::string& outContent); + void reset() { + currentState = State::Content; + currentFunction.clear(); + currentParameterName.clear(); + streamContent.clear(); + lastProcessedPosition = 0; + toolCallPositions = ToolCallPositions{}; + } State getCurrentState() const { return this->currentState; } @@ -138,49 +144,37 @@ class OnyxToolParser : public BaseOutputParser { static const std::string END_OF_TURN_TAG; // "<|eot|>" private: - const ToolsSchemas_t& toolSchemas; // filled outside; kept as reference (may change) + const ToolsSchemas_t& toolSchemas; ToolsParameterTypeMap_t toolsParametersTypes; - bool filledParametersTypesMap{false}; OnyxToolParserImpl streamParser; int toolCallIndex{-1}; std::set returnedFirstDeltas; std::set returnedCompleteDeltas; - // Mutable because it is lazily (re)built from toolSchemas inside the const getParsingStartTags() - // getter below. toolSchemas is a reference that is empty at construction time and only filled in - // by the caller afterwards (once the request's tools are known), so building this list once in the - // constructor would permanently miss every "to=" entry. Rebuilding it from scratch on every - // call keeps it in sync with whatever tools the current request declares, without hardcoding any - // tool name. - mutable std::vector parsingStartTags; - - std::optional sendFirstDeltaIfNeeded(const std::string& functionName); - std::optional sendFullDelta(const ToolCalls_t& toolCalls); - void lazyFillInitToolParametersTypesMap(); - void lazyFillParsingStartTags() const; + + std::optional sendFirstDeltaIfNeeded(const std::string& functionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + void buildStartTags(); public: OnyxToolParser() = delete; - explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas); - - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - lazyFillParsingStartTags(); - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; + explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas, + std::optional configOverride = std::nullopt); + + void resetState() override { + streamParser.reset(); + toolCallIndex = -1; + returnedFirstDeltas.clear(); + returnedCompleteDeltas.clear(); } - const std::string& getParsingEndTag() const override { - return TOOL_END_TAG; - } - bool requiresStreamingWithSpecialTokens() const override { - return true; - } - const std::vector& getSpecialTagsToErase() const override { - static const std::vector specialTagsToErase{ASSISTANT_PREFIX, CONTENT_START_INDICATOR, MESSAGE_TAG, END_OF_TURN_TAG}; - return specialTagsToErase; + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {""}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + return cfg; } }; } // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 4c80ab2b1a..b0f8f86fe4 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -32,11 +32,12 @@ #include "gemma4/gemma4_reasoning_parser.hpp" #include "gptoss/reasoning_parser.hpp" #include "lfm2/lfm2_tool_parser.hpp" -#include "lfm2/lfm25_tool_parser.hpp" #include "lfm2/lfm25_reasoning_parser.hpp" #include "gemma4/gemma4_tool_parser.hpp" #include "onyx/onyx_tool_parser.hpp" #include "onyx/onyx_reasoning_parser.hpp" +#include "onyx/onyx_content_parser.hpp" +#include "default_content_parser.hpp" #include "minicpm5/minicpm5_tool_parser.hpp" #include "minicpm5/minicpm5_reasoning_parser.hpp" @@ -116,56 +117,34 @@ const std::string& OutputParser::StreamOutputCache::getBuffer() const { return buffer; } -// TODO: @przepeck We should consider moving this and -// similar workarounds to a content parser class -static void eraseTagsFromContent(std::string& content, const std::vector& tags) { - for (const auto& tag : tags) { - size_t pos = 0; - while ((pos = content.find(tag, pos)) != std::string::npos) { - content.erase(pos, tag.length()); - } - } -} - -std::optional OutputParser::parseContentChunk(ProcessingPhase newPhase) { - std::string chunkContent = streamOutputCache.getBuffer(); - if (toolParser != nullptr) { - auto& tagsToErase = toolParser->getSpecialTagsToErase(); - auto lookupResult = streamOutputCache.lookupTags(tagsToErase); - if (lookupResult == TagLookupStatus::FOUND_COMPLETE) { - eraseTagsFromContent(chunkContent, tagsToErase); - } else if (lookupResult == TagLookupStatus::FOUND_INCOMPLETE) { - return std::nullopt; - } - } - - if (chunkContent.empty() || chunkContent == "") { - streamOutputCache.clear(); - processingPhase = newPhase; - return std::nullopt; - } - - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("content"); - writer.String(chunkContent.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); +std::optional OutputParser::parseContentChunk(ProcessingPhase newPhase) { + auto result = contentParser->parseChunk(streamOutputCache.getBuffer(), {}, ov::genai::GenerationFinishReason::NONE); + if (!result.has_value()) + return std::nullopt; // hold — keep buffer streamOutputCache.clear(); processingPhase = newPhase; - return doc; + // Suppress preamble-only ContentDelta (empty text = structural tag consumed, nothing to emit). + if (const auto* cd = std::get_if(&*result)) { + if (cd->text.empty()) + return std::nullopt; + } + return result; } -std::optional OutputParser::parseToolCallChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { +std::optional OutputParser::parseToolCallChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { if (!toolParser) { throw std::runtime_error("Tool parser is not available, cannot parse tool call chunk"); } - std::optional result; + // Bytes after the end tag belong to the next phase — preserve them before clearing. + std::string remainder; + const std::string& endTag = toolParser->getParsingConfig().endTag; + if (!endTag.empty()) { + const std::string& buf = streamOutputCache.getBuffer(); + const size_t pos = buf.find(endTag); + if (pos != std::string::npos) + remainder = buf.substr(pos + endTag.size()); + } + std::optional result; try { result = toolParser->parseChunk(streamOutputCache.getBuffer(), tokens, finishReason); } catch (...) { @@ -174,14 +153,25 @@ std::optional OutputParser::parseToolCallChunk(const std::v } streamOutputCache.clear(); processingPhase = newPhase; + if (!remainder.empty()) + streamOutputCache.add(remainder); return result; } -std::optional OutputParser::parseReasoningChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { +std::optional OutputParser::parseReasoningChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase) { if (!reasoningParser) { throw std::runtime_error("Reasoning parser is not available, cannot parse reasoning chunk"); } - std::optional result; + // Bytes after the end tag belong to the next phase — preserve them before clearing. + std::string remainder; + const std::string& endTag = reasoningParser->getParsingConfig().endTag; + if (!endTag.empty()) { + const std::string& buf = streamOutputCache.getBuffer(); + const size_t pos = buf.find(endTag); + if (pos != std::string::npos) + remainder = buf.substr(pos + endTag.size()); + } + std::optional result; try { result = reasoningParser->parseChunk(streamOutputCache.getBuffer(), tokens, finishReason); } catch (...) { @@ -190,6 +180,8 @@ std::optional OutputParser::parseReasoningChunk(const std:: } streamOutputCache.clear(); processingPhase = newPhase; + if (!remainder.empty()) + streamOutputCache.add(remainder); return result; } @@ -210,16 +202,7 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } else if (toolParserName == "devstral") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "lfm2") { - auto vocab = tokenizer.get_vocab(); - auto token = vocab.find(Lfm25ToolParser::TOOL_CALL_START_TAG); - auto tokenId = token != vocab.end() ? token->second : -1; - if (tokenId == Lfm25ToolParser::toolCallStartTokenId) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Using Lfm25ToolParser for tool parsing"); - toolParser = std::make_unique(tokenizer); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Using Lfm2ToolParser for tool parsing"); - toolParser = std::make_unique(tokenizer); - } + toolParser = std::make_unique(tokenizer); } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); } else if (toolParserName == "onyx") { @@ -243,18 +226,45 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "onyx") { reasoningParser = std::make_unique(tokenizer); - decodeWithSpecialTokens = true; } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); } - // TODO: To be considered: If we still need this check after introduction of OvmsTextStreamer. - if (toolParser && reasoningParser) { - if (toolParser->requiresStreamingWithSpecialTokens() != reasoningParser->requiresStreamingWithSpecialTokens()) { - throw std::runtime_error("Cannot use tool parser " + toolParserName + " with reasoning parser " + reasoningParserName + - " as they have different requirements for special tokens in streaming mode"); - } + // Model/output formats whose structural tokens must stay visible in the content/unknown phase + // (e.g. GptOss uses <|channel|>... throughout the stream; devstral's [TOOL_CALLS] tag and + // minicpm5's /<|im_end|> must be visible before parser-owned phases begin). For all other + // parser combinations the content phase decodes with skip_special_tokens=true (the default, + // lower noise). Each parser that requires this sets defaultDecodingWithSpecialTokens in its config. + if (toolParserName == "onyx" || reasoningParserName == "onyx") + contentParser = std::make_unique(tokenizer); + else if (toolParserName == "gptoss" || reasoningParserName == "gptoss") + contentParser = std::make_unique(tokenizer, std::vector{ + "<|start|>assistant<|channel|>final<|message|>", + "<|channel|>final<|message|>", + "<|channel|>commentary<|message|>", + "<|end|>", + "<|return|>"}); + else if (toolParserName == "gemma4") + contentParser = std::make_unique(tokenizer, std::vector{"", "<|tool_response>"}); + else if (toolParserName == "lfm2") + contentParser = std::make_unique(tokenizer, std::vector{"<|im_end|>"}); + else if (toolParserName == "minicpm5") + contentParser = std::make_unique(tokenizer, std::vector{"", "<|im_end|>"}); + else + contentParser = std::make_unique(tokenizer); + + defaultDecodingWithSpecialTokens = + (toolParser && toolParser->getParsingConfig().defaultDecodingWithSpecialTokens) || + (reasoningParser && reasoningParser->getParsingConfig().defaultDecodingWithSpecialTokens); + + if (llm_calculator_logger->should_log(spdlog::level::debug)) { + std::string toolParsingConfigStr = toolParser ? toolParser->buildParsingConfigStringRepresentation() : "N/A"; + std::string reasoningParsingConfigStr = reasoningParser ? reasoningParser->buildParsingConfigStringRepresentation() : "N/A"; + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, + "OutputParser initialized with tool parser: \"{}\" (parsing config: {}), reasoning parser: \"{}\" (parsing config: {}), defaultDecodingWithSpecialTokens={}", + toolParserName, toolParsingConfigStr, reasoningParserName, reasoningParsingConfigStr, + defaultDecodingWithSpecialTokens); } } @@ -268,13 +278,65 @@ bool OutputParser::isReasoningParserAvailable() const { std::string OutputParser::getToolParserStartTag() const { if (toolParser) { - return toolParser->getParsingStartTags()[0]; + return toolParser->getParsingConfig().startTags[0]; } else { throw std::runtime_error("Tool parser is not available, cannot get start tag"); } } +void OutputParser::resetStreamingState() { + processingPhase = UNKNOWN; + streamOutputCache.clear(); + if (toolParser) + toolParser->resetState(); + if (reasoningParser) + reasoningParser->resetState(); + if (contentParser) + contentParser->resetState(); + if (implicitReasoningStart) { + setImplicitReasoningStart(true); + } +} + +bool OutputParser::needSpecialTokensForCurrentDecode(bool userWantsSpecialTokens) const { + // Content / unknown phase: use the computed baseline for this parser combination; + // also honour user preference here (scoped to content — does not override parser phases). + if (processingPhase == CONTENT || processingPhase == UNKNOWN) { + return defaultDecodingWithSpecialTokens || userWantsSpecialTokens; + } + if (processingPhase == REASONING) { + return reasoningParser && reasoningParser->getParsingConfig().needsSpecialTokens; + } + if (processingPhase == TOOL_CALLS_PROCESSING_TOOL || processingPhase == TOOL_CALLS_WAITING_FOR_TOOL) { + return toolParser && toolParser->getParsingConfig().needsSpecialTokens; + } + return false; +} + +std::string OutputParser::getPhaseStartTagForToken(int64_t tokenId, bool toolsAvailable) const { + // The guard conditions mirror isPhaseStartToken: don't re-fire for a phase we are + // already in (the parser's own text-based detection handles re-entry there). + if (toolParser && toolsAvailable) { + const auto& tokenMap = toolParser->getResolvedStartTokenToTag(); + auto it = tokenMap.find(tokenId); + if (it != tokenMap.end() && + processingPhase != TOOL_CALLS_PROCESSING_TOOL && + processingPhase != TOOL_CALLS_WAITING_FOR_TOOL) { + return it->second; + } + } + if (reasoningParser) { + const auto& tokenMap = reasoningParser->getResolvedStartTokenToTag(); + auto it = tokenMap.find(tokenId); + if (it != tokenMap.end() && processingPhase != REASONING) { + return it->second; + } + } + return {}; +} + void OutputParser::setImplicitReasoningStart(bool value) { + implicitReasoningStart = value; if (!reasoningParser) { return; } @@ -293,34 +355,14 @@ void OutputParser::detectAndSetImplicitReasoningStart(const std::string& rendere } std::string trimmed = renderedPrompt; rtrim(trimmed); - const auto& startTags = reasoningParser->getParsingStartTags(); + const auto& startTags = reasoningParser->getParsingConfig().startTags; bool detected = std::any_of(startTags.begin(), startTags.end(), [&](const std::string& tag) { return !tag.empty() && endsWith(trimmed, tag); }); setImplicitReasoningStart(detected); return; } -ParsedOutput OutputParser::parse(const std::vector& generatedTokens, const bool toolsAvailable) { - // Model output is processed by the chain of parsers. Each parser extracts relevant part of the output and fills the ParsedOutput structure. - // At the beginning, the content field of ParsedOutput is already filled with decoded content from generatedTokens. - // When parser extracts relevant information, it should remove it from the content field, so we don't duplicate it in the final output. - - if (spdlog::default_logger_raw()->level() == spdlog::level::trace) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); - } - ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(!decodeWithSpecialTokens)); - if (reasoningParser) { - reasoningParser->parse(parsedOutput, generatedTokens); - } - // We run tool parser only if the parser is available and tools have been provided in the request. - if (toolParser && toolsAvailable) { - toolParser->parse(parsedOutput, generatedTokens); - } - return parsedOutput; -} - -std::optional OutputParser::parseChunk(const std::string& chunkResponse, const std::vector& tokens, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { +std::optional OutputParser::parseChunk(const std::string& chunkResponse, const std::vector& tokens, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { /* Using appropriate parser based on the current processing phase Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase @@ -329,21 +371,57 @@ std::optional OutputParser::parseChunk(const std::string& c so only use those methods or return nullopt. */ - bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingStartTags().empty() && !reasoningParser->getParsingEndTag().empty(); - bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingStartTags().empty(); + bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingConfig().startTags.empty() && !reasoningParser->getParsingConfig().endTag.empty(); + bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingConfig().startTags.empty(); bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable; streamOutputCache.add(chunkResponse); + if (llm_calculator_logger->should_log(spdlog::level::trace)) { + std::string tokenIds; + tokenIds.reserve(tokens.size() * 7); + for (size_t i = 0; i < tokens.size(); ++i) { + if (i > 0) + tokenIds += ", "; + tokenIds += std::to_string(tokens[i]); + } + + std::string processingPhaseStr; + switch (processingPhase) { + case UNKNOWN: + processingPhaseStr = "UNKNOWN"; + break; + case CONTENT: + processingPhaseStr = "CONTENT"; + break; + case REASONING: + processingPhaseStr = "REASONING"; + break; + case TOOL_CALLS_PROCESSING_TOOL: + processingPhaseStr = "TOOL_CALLS_PROCESSING_TOOL"; + break; + case TOOL_CALLS_WAITING_FOR_TOOL: + processingPhaseStr = "TOOL_CALLS_WAITING_FOR_TOOL"; + break; + default: + processingPhaseStr = "UNKNOWN"; + break; + } + + SPDLOG_LOGGER_TRACE(llm_calculator_logger, + "OutputParser::parseChunk[PROCESSING_PHASE={}] called with {} tokens, text=\"{}\", finish_reason={}, token IDs=[{}]", + processingPhaseStr, tokens.size(), chunkResponse, static_cast(finishReason), tokenIds); + } + if (processingPhase == UNKNOWN) { // If we are in the UNKNOWN phase, we need to determine if we should switch to CONTENT, REASONING, or TOOL_CALLS phase. TagLookupStatus anyStartTagStatus = TagLookupStatus::NOT_FOUND; if (reasoningParserExistsAndSupportsStreaming) { // Check if reasoning start tag has been received - TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingStartTags()); + TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingConfig().startTags); if (reasoningStartTagStatus == TagLookupStatus::NOT_FOUND) { // If reasoning start tag is not found, check if any of the special start tags are found - reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getSpecialParsingStartTags()); + reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingConfig().preambleStartTags); } if (reasoningStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseReasoningChunk(tokens, finishReason); @@ -353,10 +431,10 @@ std::optional OutputParser::parseChunk(const std::string& c if (applyToolParser) { // Check if tool call start tag has been received - TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().startTags); if (toolCallStartTagStatus == TagLookupStatus::NOT_FOUND) { // If tool call start tag is not found, check if any of the special start tags are found - toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getSpecialParsingStartTags()); + toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().preambleStartTags); } if (toolCallStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseToolCallChunk(tokens, finishReason); @@ -374,7 +452,7 @@ std::optional OutputParser::parseChunk(const std::string& c return std::nullopt; } else if (processingPhase == REASONING) { // If we are in the REASONING phase, we check if parsing end tag is found and if so, switch to UNKNOWN phase. - TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingEndTag()); + TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingConfig().endTag); if (endTagStatus == TagLookupStatus::FOUND_COMPLETE) { // Switch back to UNKNOWN phase (we can have either CONTENT or TOOL_CALLS next) return parseReasoningChunk(tokens, finishReason, UNKNOWN); @@ -386,7 +464,7 @@ std::optional OutputParser::parseChunk(const std::string& c // If we are in the CONTENT phase, we check if tool parser start tag is found and if so, switch to TOOL_CALLS phase. // TOOL_CALLS is the only phase that can be processed after CONTENT. if (applyToolParser) { - TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); + TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().startTags); if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseToolCallChunk(tokens, finishReason); } else if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { @@ -396,29 +474,36 @@ std::optional OutputParser::parseChunk(const std::string& c } return parseContentChunk(); } else if (processingPhase == TOOL_CALLS_PROCESSING_TOOL) { - // Processing TOOL_CALLS is the last phase, so we always return the result of tool parser. - TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingEndTag()); + // Active tool call: accumulate until the end tag, then transition to WAITING_FOR_TOOL + // to determine whether another tool call or a content turn follows. + TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingConfig().endTag); if (toolEndTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { return std::nullopt; // Wait for more chunks to determine if end tag is complete } if (toolEndTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // If tool call has finished, we switch to waiting for next tool call as tool calls in the last phase, - // so we either get next tool call or finish processing. return parseToolCallChunk(tokens, finishReason, TOOL_CALLS_WAITING_FOR_TOOL); } return parseToolCallChunk(tokens, finishReason); } else if (processingPhase == TOOL_CALLS_WAITING_FOR_TOOL) { - // In this phase we are waiting for next tool call or finish of generation. - // If we get next tool call start tag, we switch to TOOL_CALLS phase, otherwise if generation finishes we switch to CONTENT phase to flush any remaining content. - TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingStartTags()); - if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if start tag is complete - } + TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().startTags); if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // If tool call has started, we switch back to processing tool phase. return parseToolCallChunk(tokens, finishReason, TOOL_CALLS_PROCESSING_TOOL); } - return parseToolCallChunk(tokens, finishReason); + const auto& contentTurnStartTags = contentParser->getParsingConfig().startTags; + if (!contentTurnStartTags.empty()) { + TagLookupStatus contentTurnStatus = streamOutputCache.lookupTags(contentTurnStartTags); + if (contentTurnStatus == TagLookupStatus::FOUND_COMPLETE) { + return parseContentChunk(); + } + if (finishReason != ov::genai::GenerationFinishReason::NONE) { + return parseContentChunk(); + } + return std::nullopt; + } + if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; + } + return parseToolCallChunk(tokens, finishReason, TOOL_CALLS_WAITING_FOR_TOOL); } else { SPDLOG_LOGGER_ERROR(llm_calculator_logger, "Unexpected processing phase: {}", static_cast(processingPhase)); throw std::runtime_error("Unexpected error during stream output parsing"); diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index 991cd4902a..47d4338f21 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -26,6 +26,34 @@ namespace ovms { +// OutputParser orchestrates the streaming parsing pipeline. +// +// Responsibilities of OutputParser (the orchestrator): +// - Phase lifecycle: detect phase transitions by looking for start/end tags declared +// in each parser's OutputParsingConfig; switch the active phase accordingly. +// - Buffer management: accumulate decoded text in StreamOutputCache, hold it while a +// tag is only partially matched, flush it to the active parser when a boundary is +// confirmed, and carry over any bytes that trail a phase-end tag so they seed the +// next phase without loss. +// - Routing: deliver each flush exclusively to the parser that owns the current phase. +// A specific parser's parseChunk() is only ever called with text that belongs to its +// active phase — the parser does not need to detect or handle phase transitions. +// - Coordination: manage the interplay between tool, reasoning, and content parsers +// across the full generation sequence. +// +// Responsibilities of BaseOutputParser subclasses (the specific parsers): +// - Declare phase boundaries by returning a correctly populated OutputParsingConfig +// (startTags, endTag, preambleStartTags, etc.). This is the sole coupling point +// with the orchestrator — no knowledge of OutputParser internals is required. +// - Implement parseChunk() to process the text it receives during its active phase. +// The parser may maintain arbitrary internal state and buffers to satisfy its own +// format requirements; OutputParser does not inspect or constrain that state. +// - Return a JSON delta (OpenAI streaming format) or nullopt to signal "nothing to +// emit yet"; the orchestrator propagates that decision upstream unchanged. +// +// Design invariant: OutputParser must contain NO logic specific to any individual model +// format. All format-specific behaviour must be encapsulated in the parser subclasses +// and expressed through their configuration and parseChunk() implementations. class OutputParser { // Public types and enums public: @@ -56,21 +84,27 @@ class OutputParser { private: ov::genai::Tokenizer tokenizer; - std::unique_ptr toolParser = nullptr; // Tool parser for extracting tool calls - std::unique_ptr reasoningParser = nullptr; // Reasoning parser for extracting reasoning content - bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) + std::unique_ptr toolParser = nullptr; + std::unique_ptr reasoningParser = nullptr; + std::unique_ptr contentParser = nullptr; // Streaming related members ProcessingPhase processingPhase = UNKNOWN; StreamOutputCache streamOutputCache; + bool implicitReasoningStart = false; + + // Baseline decode mode for content/unknown phases — true when the model/output format + // needs special tokens visible before any parser-owned phase becomes active. + // Set once in the constructor from parser names. + bool defaultDecodingWithSpecialTokens = false; // Parsing methods below read chunks from streamOutputCache hence no string argument is needed // Regular content parsing method does not require finishReason as content is always parsed - std::optional parseContentChunk(ProcessingPhase newPhase = CONTENT); + std::optional parseContentChunk(ProcessingPhase newPhase = CONTENT); - std::optional parseToolCallChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase = TOOL_CALLS_PROCESSING_TOOL); - std::optional parseReasoningChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase = REASONING); + std::optional parseToolCallChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase = TOOL_CALLS_PROCESSING_TOOL); + std::optional parseReasoningChunk(const std::vector& tokens, ov::genai::GenerationFinishReason finishReason, ProcessingPhase newPhase = REASONING); // Configure parser to treat the output as already-in-reasoning from the first token. // Used when the chat template appends the reasoning start tag (e.g. "\n") as @@ -86,25 +120,49 @@ class OutputParser { bool isReasoningParserAvailable() const; std::string getToolParserStartTag() const; + // Reset streaming state (phase, buffer, and per-parser internal state) for a new generation. + void resetStreamingState(); + // Auto-detect and apply implicit reasoning start based on the prompt produced by the chat template. void detectAndSetImplicitReasoningStart(const std::string& renderedPrompt); - // Parse model output in the unary mode. Returns ParsedOutput containing data extracted by internal parsers. - ParsedOutput parse(const std::vector& generatedTokens, const bool toolsAvailable); - - // Parse model output chunk in the steaming mode. Returns a JSON object containing the delta that conforms to OpenAI API - // or nullopt if no response can be produced. - // tokens holds the token IDs that produced chunkResponse (may be empty; currently informational for future use). - std::optional parseChunk(const std::string& chunkResponse, const std::vector& tokens, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason); - - bool requiresStreamingWithSpecialTokens() const { - if (!reasoningParser) { - return toolParser && toolParser->requiresStreamingWithSpecialTokens(); - } else if (!toolParser) { - return reasoningParser && reasoningParser->requiresStreamingWithSpecialTokens(); - } else { - return (reasoningParser && reasoningParser->requiresStreamingWithSpecialTokens()) && (toolParser && toolParser->requiresStreamingWithSpecialTokens()); - } - } + // Parse one decoded chunk in streaming mode. + // + // Contract: + // - Returns a JSON delta conforming to the OpenAI streaming API, or nullopt when no + // output can yet be produced (partial tag match, preamble stripping, etc.). + // - Processes AT MOST ONE phase per call. If a chunk spans a phase boundary (e.g. a + // token whose text contains both an end tag and the start of the next phase), the bytes + // after the end tag are preserved in the internal buffer and processed on the next call. + // No content is ever discarded at a phase transition — "nothing properly parsed is lost". + // - Correctness requires at least one subsequent call after every phase transition. + // The caller must provide a final call with finishReason != NONE (typically an empty + // chunk) so the buffer is fully drained. If that call is missing, any buffered + // remainder from the last transition will be silently dropped. + // - Known limitation: if finishReason != NONE arrives while a start tag is only + // partially matched (FOUND_INCOMPLETE), the partial text is flushed as content rather + // than held for completion — this is unavoidable without more tokens. + // + // Implementation must be fully generic: no parser-specific logic or special-casing of + // individual model formats belongs here. Behaviour must be driven entirely by the + // configuration exposed through BaseOutputParser::getParsingConfig(). + // + // tokens holds the token IDs that produced chunkResponse (informational; used for + // token-ID-based phase-start detection in OVMSTextStreamer). + std::optional parseChunk(const std::string& chunkResponse, const std::vector& tokens, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason); + + // Decide decode mode dynamically based on parser phase and user preference. + // Content/unknown phases use defaultDecodingWithSpecialTokens OR user preference. + // Reasoning/tool phases are driven solely by the active parser's needsSpecialTokens flag; + // user preference does not override parser correctness requirements in those phases. + bool needSpecialTokensForCurrentDecode(bool userWantsSpecialTokens = false) const; + + // If `tokenId` is a phase-start token, returns the corresponding tag string + // (taken directly from resolvedStartTokenToTag — no tokenizer decode needed). + // Tool phase-start tags are considered only when toolsAvailable is true. + // Returns an empty string when the token is not a known phase-start token. + // Used by OVMSTextStreamer to immediately flush the start-tag text without + // going through the delay buffer. + std::string getPhaseStartTagForToken(int64_t tokenId, bool toolsAvailable = true) const; }; } // namespace ovms diff --git a/src/llm/io_processing/output_parsing_config.hpp b/src/llm/io_processing/output_parsing_config.hpp new file mode 100644 index 0000000000..bc1dd99ffa --- /dev/null +++ b/src/llm/io_processing/output_parsing_config.hpp @@ -0,0 +1,74 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +namespace ovms { + +// Configuration for a parser's phase-boundary detection and tokenizer decode mode. +// +// Phase detection: +// startTags — text-based start-boundary strings (matched against decoded text). +// tokenIdStartTags — start-boundary strings that are single special tokens. +// On construction the base class encodes each via the tokenizer +// (add_special_tokens=false). If the encoding resolves to exactly +// one token the ID is stored in resolvedStartTokenToTag and checked +// against incoming token IDs BEFORE any string matching. +// A start-tag token detected this way is synthesised as text before +// being passed to parseChunk(), so the sub-parser state machine +// receives the expected boundary string. +// preambleStartTags — text-based tags checked only in the UNKNOWN (preamble) phase. +// These are alternative entry points that cannot appear mid-stream. +// endTag — text-based end-boundary string (checked in TOOL_CALLS_PROCESSING_TOOL +// and REASONING phases). +// stringsToErase — strings stripped from this parser's output before emission +// (e.g. BOS/EOS tokens that leak due to special-token decode mode, +// or chat-template structural markers). +// +// Tokenizer decode mode flag (evaluated by OutputParser::needSpecialTokensForCurrentDecode): +// needsSpecialTokens — Decode with skip_special_tokens=false while this parser is in its +// active phase (REASONING for reasoning parsers; TOOL_CALLS_* for tool +// parsers). The parser's internal state machine relies on special-token +// strings being visible in the decoded text during that phase. +// +// Parsers that detect phase boundaries via token IDs only (Llama3, Hermes3, Phi4, Mistral, +// Qwen3, Qwen3Coder) leave this flag false — the proactive token-ID switch in OVMSTextStreamer +// synthesises the start-tag text without requiring special-token decode in the active phase. +// +// Whether the content/unknown phase also needs special tokens is determined at the +// OutputParser level via defaultDecodingWithSpecialTokens, not in the per-parser config. +// +// Content/unknown phase decode mode: +// defaultDecodingWithSpecialTokens — when true, decoding uses skip_special_tokens=false +// even in the content/unknown phase. Set by parsers +// whose model format emits structural special tokens +// before their own active phase begins (e.g. GptOss, +// devstral, minicpm5). +struct OutputParsingConfig { + std::vector startTags; + std::vector tokenIdStartTags; + std::vector preambleStartTags; + std::string endTag; + std::vector stringsToErase; + + bool needsSpecialTokens = false; + // See comment block above. + bool defaultDecodingWithSpecialTokens = false; +}; + +} // namespace ovms diff --git a/src/llm/io_processing/phi4/tool_parser.cpp b/src/llm/io_processing/phi4/tool_parser.cpp index a9e6366026..25636ae2fc 100644 --- a/src/llm/io_processing/phi4/tool_parser.cpp +++ b/src/llm/io_processing/phi4/tool_parser.cpp @@ -109,56 +109,7 @@ void Phi4ToolParser::clearState() { openBracesCount = 1; // Reset to 1 as we count the tool call opening brace } -void Phi4ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - std::vector tools; - - // Phi4 with vLLM template produces tool calls in the format: - // functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...] - - std::string toolsStartString = "functools"; - size_t toolsStartPos = 0; - toolsStartPos = parsedOutput.content.find(toolsStartString); - - if (toolsStartPos != std::string::npos) { - // Extract the tools part, assuming it's all the remaining content after "functools" - std::string toolsString = parsedOutput.content.substr(toolsStartPos + toolsStartString.length()); - rapidjson::Document toolsDoc; - toolsDoc.Parse(toolsString.c_str()); - if (!toolsDoc.HasParseError() && toolsDoc.IsArray()) { - for (auto& toolVal : toolsDoc.GetArray()) { - if (!toolVal.IsObject()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call is not a valid JSON object"); - continue; - } - ToolCall toolCall; - toolCall.id = generateRandomId(); // Generate a random ID for the tool call - if (toolVal.HasMember("name") && toolVal["name"].IsString()) { - toolCall.name = toolVal["name"].GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid name field"); - continue; - } - - if (toolVal.HasMember("arguments") && toolVal["arguments"].IsObject()) { - rapidjson::StringBuffer sb; - rapidjson::Writer toolWriter(sb); - toolVal["arguments"].Accept(toolWriter); - toolCall.arguments = sb.GetString(); - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Tool call does not contain valid parameters object"); - continue; - } - parsedOutput.toolCalls.push_back(toolCall); - } - } else { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to parse functools content or extract tools array"); - } - // Remove the tools part from the content - parsedOutput.content.erase(toolsStartPos); - } -} - -std::optional Phi4ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional Phi4ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { /* Phi4 with vLLM template produces tool calls in the format: functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...] @@ -285,7 +236,6 @@ std::optional Phi4ToolParser::parseChunk(const std::string& throw std::runtime_error("Generated tool call structure is not valid"); } - rapidjson::Document doc; // Case 1: 'arguments' has just appeared in the current chunk. If so, we return first delta. if (newJson.HasMember("arguments") && !lastJson.HasMember("arguments")) { std::string functionName; @@ -299,9 +249,8 @@ std::optional Phi4ToolParser::parseChunk(const std::string& throw std::runtime_error("Tool call name is missing in generated output"); } // Wrap first delta in {"tool_calls":[{"id":,"type":"function","index":,"function":{"name": }}]} - doc = wrapFirstDelta(functionName, toolCallIndex); lastJson.CopyFrom(newJson, lastJson.GetAllocator()); - return doc; + return ToolCallDelta{toolCallIndex, generateRandomId(), functionName, ""}; // Case 2: 'arguments' already exists in the last JSON, we compute delta and return it. } else if (lastJson.HasMember("arguments")) { rapidjson::Document delta = PartialJsonBuilder::computeDelta(lastJson, newJson); @@ -326,9 +275,11 @@ std::optional Phi4ToolParser::parseChunk(const std::string& } } - // Wrap delta in {"tool_calls":[{"index":,"function":}]} - doc = wrapDelta(delta, toolCallIndex); - return doc; + // Wrap delta in {"tool_calls":[{"index":,"function":{"arguments":"..."}}]} + std::string argsStr; + if (delta.HasMember("arguments") && delta["arguments"].IsString()) + argsStr = delta["arguments"].GetString(); + return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, argsStr}; // Case 3: No 'arguments' exists or just appeared, so we keep building up until we have complete function name } else { lastJson.CopyFrom(newJson, lastJson.GetAllocator()); diff --git a/src/llm/io_processing/phi4/tool_parser.hpp b/src/llm/io_processing/phi4/tool_parser.hpp index 7e6734d27f..008baafaff 100644 --- a/src/llm/io_processing/phi4/tool_parser.hpp +++ b/src/llm/io_processing/phi4/tool_parser.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -64,22 +65,27 @@ class Phi4ToolParser : public BaseOutputParser { public: Phi4ToolParser() = delete; - explicit Phi4ToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags = {this->parsingStartTag}; - return parsingStartTags; + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"functools"}; + return cfg; } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags = {}; - return specialParsingStartTags; - } - // Tools calls are expected to be the last part of the content, so we do not specify an end tag. - const std::string& getParsingEndTag() const override { - return parsingEndTag; + + explicit Phi4ToolParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + internalState = AWAITING_START_TAG; + lastJson.SetNull(); + jsonBuilder.clear(); + toolCallIndex = -1; + argumentsQuotesOpened = false; + unprocessedBuffer.clear(); } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/qwen3/reasoning_parser.cpp b/src/llm/io_processing/qwen3/reasoning_parser.cpp index a474c27634..c6e9d71df4 100644 --- a/src/llm/io_processing/qwen3/reasoning_parser.cpp +++ b/src/llm/io_processing/qwen3/reasoning_parser.cpp @@ -18,66 +18,47 @@ #include #include -#include "src/port/rapidjson_document.hpp" - #include "../../../logging.hpp" #include "reasoning_parser.hpp" -#include "../utils.hpp" namespace ovms { -void Qwen3ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - std::string startReasoningTag = getParsingStartTags()[0]; - std::string endReasoningTag = getParsingEndTag(); - size_t startPos = parsedOutput.content.find(startReasoningTag); - size_t endPos = parsedOutput.content.find(endReasoningTag); - // Implicit-start mode: the chat template already emitted the start tag as the prompt - // suffix, so the model output begins inside the reasoning segment. - // When active, implicit-start always takes priority - everything up to the first - // is reasoning, even if the content contains nested tags. - if (implicitStart) { - if (endPos != std::string::npos) { - parsedOutput.reasoning = parsedOutput.content.substr(0, endPos); - parsedOutput.content.erase(0, endPos + endReasoningTag.length()); - } else { - parsedOutput.reasoning = parsedOutput.content; - parsedOutput.content.clear(); - } - return; +std::optional Qwen3ReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Qwen3ReasoningParser"); + return std::nullopt; } - if (startPos != std::string::npos && endPos != std::string::npos && startPos < endPos) { - // Extract reasoning between and - size_t reasoningStart = startPos + startReasoningTag.length(); - std::string reasoningText = parsedOutput.content.substr(reasoningStart, endPos - reasoningStart); - parsedOutput.reasoning = reasoningText; - // Remove reasoning from content - parsedOutput.content.erase(startPos, endPos - startPos + endReasoningTag.length()); + // Strip the end tag and keep only the text that precedes it. + // This handles the case where the end tag token is decoded in the same + // streamer flush as preceding reasoning text (FOUND_INCOMPLETE hold-back + // accumulates e.g. "...ing" in the cache). + std::string text = chunk; + const std::string& endTag = parsingConfig.endTag; + const size_t endTagPos = text.rfind(endTag); + if (endTagPos != std::string::npos) { + text = text.substr(0, endTagPos); } -} -std::optional Qwen3ReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { - if (chunk.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Qwen3ReasoningParser"); - return std::nullopt; + // On the very first call, consume the start tag if it begins the text + // (explicit phase-entry case) or mark it consumed immediately if no start + // tag is present (implicit reasoning start — the prompt already ended with + // so the model never emits it again). + // After the first call, any that appears in the stream is literal + // reasoning content produced by the model and is emitted as-is. + if (!phaseEntryTagConsumed_) { + const std::string& startTag = parsingConfig.startTags[0]; + const size_t startTagPos = text.find(startTag); + if (startTagPos != std::string::npos) { + text = text.substr(startTagPos + startTag.size()); + } + phaseEntryTagConsumed_ = true; } - if (chunk.find(getParsingStartTags()[0]) != std::string::npos || chunk.find(getParsingEndTag()) != std::string::npos) { + if (text.empty()) { return std::nullopt; - } else { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - writer.StartObject(); - writer.String("delta"); - writer.StartObject(); - writer.String("reasoning_content"); - writer.String(chunk.c_str()); - writer.EndObject(); - writer.EndObject(); - rapidjson::Document doc; - doc.Parse(buffer.GetString()); - return doc; } - return std::nullopt; + + return ReasoningDelta{text}; } } // namespace ovms diff --git a/src/llm/io_processing/qwen3/reasoning_parser.hpp b/src/llm/io_processing/qwen3/reasoning_parser.hpp index 9b59f62760..655b157c36 100644 --- a/src/llm/io_processing/qwen3/reasoning_parser.hpp +++ b/src/llm/io_processing/qwen3/reasoning_parser.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "src/port/rapidjson_document.hpp" @@ -31,23 +32,30 @@ class Qwen3ReasoningParser : public BaseOutputParser { const std::string parsingStartTag = ""; const std::string parsingEndTag = ""; +private: + // Tracks whether the phase-entry start tag has already been consumed by parseChunk. + // On the very first call the start tag is stripped (explicit start) or skipped + // (implicit start — tag was already in the prompt). After that, any in + // the stream is treated as literal reasoning content and emitted as-is. + bool phaseEntryTagConsumed_ = false; + public: Qwen3ReasoningParser() = delete; - explicit Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} - - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{this->parsingStartTag}; - return parsingStartTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags{}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - return parsingEndTag; + + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {""}; + cfg.endTag = ""; + return cfg; } + + explicit Qwen3ReasoningParser(ov::genai::Tokenizer& tokenizer, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { phaseEntryTagConsumed_ = false; } + + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; }; } // namespace ovms diff --git a/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp b/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp index 50476c1e89..5efec57872 100644 --- a/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp +++ b/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp @@ -213,101 +213,55 @@ std::optional Qwen3CoderToolParserImpl::parseChunk(const std::strin return std::nullopt; } -void Qwen3CoderToolParser::lazyFillInitToolParametersTypesMap() { - if (this->filledParametersTypesMap) { - return; - } - SPDLOG_DEBUG("Filling tools parameters types map"); - this->toolsParametersTypes = createToolsParametersTypesMap(this->toolSchemas); - this->filledParametersTypesMap = true; - SPDLOG_DEBUG("Qwen3CoderToolParser created with {} tools", this->toolsParametersTypes.size()); -} - -Qwen3CoderToolParser::Qwen3CoderToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : - BaseOutputParser(tokenizer), +Qwen3CoderToolParser::Qwen3CoderToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas, + std::optional configOverride) : + BaseOutputParser(tokenizer, [&]() { + if (configOverride.has_value()) + return std::move(*configOverride); + OutputParsingConfig cfg; + cfg.startTags = {TOOL_START_TAG, FUNCTION_NAME_TAG}; + return cfg; + }()), toolSchemas(toolSchemas), + toolsParametersTypes(createToolsParametersTypesMap(toolSchemas)), streamParser(this->toolsParametersTypes) { } -void Qwen3CoderToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // there may be multiple parameters per function, - // there may be multiple lines per parameter value - // there may be no parameters for a function - // there may be multiple tool_call sections in the content - // there is only one function per tool call - // - // - // - // PARAM_VALUE - // - // - // - this->lazyFillInitToolParametersTypesMap(); - auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); - if (toolCallsOpt.has_value()) { - // TODO do we want to support not ending in content state? - parsedOutput.toolCalls = std::move(toolCallsOpt.value()); - SPDLOG_DEBUG("Parsing ended successfully, removing tool calls from content"); - auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); - if (!status.ok()) { - SPDLOG_DEBUG("Failed to remove tool calls from content: {}", status.string()); - } - return; - } - SPDLOG_DEBUG("Parsing ended, no tool calls found"); - return; -} std::optional Qwen3CoderToolParserImpl::getCurrentFunctionName() const { if (this->currentFunction.name.empty()) { return std::nullopt; } return this->currentFunction.name; } -std::optional Qwen3CoderToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { +std::optional Qwen3CoderToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { if (toolCalls.size() != 1) { SPDLOG_ERROR("For streaming we expected one tool call, got: {}", toolCalls.size()); - // TODO we should return status code but this require change of parsers API throw std::runtime_error("For streaming we expected one tool call"); } auto& toolCall = toolCalls[0]; - rapidjson::Document argsDelta; - argsDelta.Parse(toolCall.arguments.c_str()); this->returnedCompleteDeltas.insert(this->toolCallIndex); - rapidjson::Document argumentsWrapper; - argumentsWrapper.SetObject(); - rapidjson::Document::AllocatorType& allocator = argumentsWrapper.GetAllocator(); - // now we need to add string toolCall.arguments to argumentsWrapper under "arguments" key - rapidjson::Value toolCallsString(rapidjson::kStringType); - toolCallsString.SetString(toolCall.arguments.c_str(), allocator); SPDLOG_TRACE("Tool call arguments string: {}", toolCall.arguments); - - argumentsWrapper.AddMember("arguments", toolCallsString, allocator); - auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); - SPDLOG_DEBUG("First delta doc: {}", documentToString(currentDelta)); - return currentDelta; + SPDLOG_DEBUG("Full delta: index={} arguments={}", this->toolCallIndex, toolCall.arguments); + return ToolCallDelta{this->toolCallIndex, std::nullopt, std::nullopt, toolCall.arguments}; } -std::optional Qwen3CoderToolParser::sendFirstDeltaIfNeeded(const std::string& toolCallName) { +std::optional Qwen3CoderToolParser::sendFirstDeltaIfNeeded(const std::string& toolCallName) { if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { SPDLOG_TRACE("Skipping first delta, already sent for current function, returnedFirstDeltas.size(): {} returnedCompleteDeltas.size(): {}", returnedFirstDeltas.size(), returnedCompleteDeltas.size()); - // we can skip sending first delta since we sent it for current function return std::nullopt; } int toolCallId = ++this->toolCallIndex; - rapidjson::Document doc = wrapFirstDelta(toolCallName, toolCallId); - this->currentJson.CopyFrom(doc, this->currentJson.GetAllocator()); this->returnedFirstDeltas.insert(toolCallId); - SPDLOG_DEBUG("First delta doc: {}", documentToString(doc)); - return doc; + SPDLOG_DEBUG("First delta: name={} index={}", toolCallName, toolCallId); + return ToolCallDelta{toolCallId, generateRandomId(), toolCallName, ""}; } -std::optional Qwen3CoderToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional Qwen3CoderToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { // streamParser will return optional toolCalls when a tool call is completed // if toolCalls is returned, we need to wrap it in the required JSON structure and return it // if toolCalls is not returned, but we are insideFunction state, we need to return the first delta with function name once // otherwise nullopt SPDLOG_DEBUG("Chunk: '{}', finishReason: {}", newChunk, static_cast(finishReason)); - this->lazyFillInitToolParametersTypesMap(); if (newChunk.empty()) { return std::nullopt; } diff --git a/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.hpp b/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.hpp index b5db9c019d..73eaf4d301 100644 --- a/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.hpp +++ b/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.hpp @@ -88,6 +88,14 @@ C->ITC->IFN->IF->IPN->IP->AF->C std::optional parseChunk(const std::string& chunk); std::optional getCurrentFunctionName() const; Status removeToolCallsFromContentIfNeeded(std::string& outContent); + void reset() { + currentState = State::Content; + currentFunction.clear(); + currentParameterName.clear(); + streamContent.clear(); + lastProcessedPosition = 0; + toolCallPositions = ToolCallPositions{}; + } State getCurrentState() const { return this->currentState; } @@ -130,40 +138,30 @@ class Qwen3CoderToolParser : public BaseOutputParser { static const std::string XML_TAG_END; private: - const ToolsSchemas_t& toolSchemas; // we need to keep reference as this is not filled in OpenAIApiHandler during ToolParser creation, NOTE that its const here but it can change outside + const ToolsSchemas_t& toolSchemas; ToolsParameterTypeMap_t toolsParametersTypes; - bool filledParametersTypesMap{false}; // for streaming parsing we need to keep parser as a member Qwen3CoderToolParserImpl streamParser; int toolCallIndex{-1}; ToolCalls_t currentToolCalls; - rapidjson::Document currentJson; std::set returnedFirstDeltas; std::set returnedCompleteDeltas; public: Qwen3CoderToolParser() = delete; - explicit Qwen3CoderToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas); - - void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; - const std::vector& getParsingStartTags() const override { - static const std::vector startTags = {TOOL_START_TAG, FUNCTION_NAME_TAG}; - return startTags; - } - const std::vector& getSpecialParsingStartTags() const override { - static const std::vector specialParsingStartTags = {}; - return specialParsingStartTags; - } - const std::string& getParsingEndTag() const override { - static const std::string EMPTY_STRING = ""; - return EMPTY_STRING; + explicit Qwen3CoderToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas, + std::optional configOverride = std::nullopt); + void resetState() override { + streamParser.reset(); + toolCallIndex = -1; + returnedFirstDeltas.clear(); + returnedCompleteDeltas.clear(); } + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; private: - std::optional sendFirstDeltaIfNeeded(const std::string& currentFunctionName); - std::optional sendFullDelta(const ToolCalls_t& toolCalls); - void lazyFillInitToolParametersTypesMap(); + std::optional sendFirstDeltaIfNeeded(const std::string& currentFunctionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); }; } // namespace ovms template <> diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index 7c46d1a0a9..2100710a27 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -15,6 +15,7 @@ //***************************************************************************** #include #include +#include #include "utils.hpp" @@ -69,6 +70,10 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str int quoteDepth = 0; int singleQuoteDepth = 0; + auto isWordChar = [](char c) { + return std::isalnum(static_cast(c)) != 0 || c == '_'; + }; + for (size_t i = startPos; i < str.length(); ++i) { if (bracketDepth == 0 && braceDepth == 0 && quoteDepth == 0 && singleQuoteDepth == 0 && str.compare(i, target.length(), target) == 0) { @@ -86,7 +91,30 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str } else if (str[i] == '"' && (i == 0 || str[i - 1] != '\\')) { quoteDepth = 1 - quoteDepth; } else if (quoteDepth == 0 && str[i] == '\'' && (i == 0 || str[i - 1] != '\\')) { - singleQuoteDepth = 1 - singleQuoteDepth; + const bool prevIsWord = (i > 0) && isWordChar(str[i - 1]); + const bool nextIsWord = (i + 1 < str.size()) && isWordChar(str[i + 1]); + + if (singleQuoteDepth == 0) { + // Opening single quote: ignore apostrophes inside words. + if (!(prevIsWord && nextIsWord)) { + singleQuoteDepth = 1; + } + } else { + // Inside single-quoted text: treat apostrophes in words as plain + // characters (it's, Johns'). Close only when the following + // non-space token looks like an argument/list/object delimiter. + if (prevIsWord && nextIsWord) { + continue; + } + + size_t j = i + 1; + while (j < str.size() && std::isspace(static_cast(str[j])) != 0) { + ++j; + } + if (j == str.size() || str[j] == ',' || str[j] == ':' || str[j] == ']' || str[j] == '}' || str[j] == ')') { + singleQuoteDepth = 0; + } + } } } return std::string::npos; diff --git a/src/llm/language_model/legacy/servable.cpp b/src/llm/language_model/legacy/servable.cpp index 4ec1e65085..dd5f130c70 100644 --- a/src/llm/language_model/legacy/servable.cpp +++ b/src/llm/language_model/legacy/servable.cpp @@ -117,13 +117,11 @@ absl::Status LegacyServable::parseRequest(std::shared_ptrapiHandler->isStream()) { - if ((legacyExecutionContext->apiHandler->getOutputParser() != nullptr && - legacyExecutionContext->apiHandler->getOutputParser()->requiresStreamingWithSpecialTokens()) || - !legacyExecutionContext->apiHandler->getRequest().skipSpecialTokens) { + { + if (!legacyExecutionContext->apiHandler->getRequest().skipSpecialTokens) { streamerConfig.insert(ov::genai::skip_special_tokens(false)); } - auto ovmsCallback = [& ctx = *legacyExecutionContext](rapidjson::Document delta, bool isLast) -> ov::genai::StreamingStatus { + auto ovmsCallback = [& ctx = *legacyExecutionContext](Delta delta, bool isLast) -> ov::genai::StreamingStatus { if (ctx.clientDisconnected.load()) { ctx.deltaChannel.signalComplete(); return ov::genai::StreamingStatus::CANCEL; @@ -137,15 +135,6 @@ absl::Status LegacyServable::parseRequest(std::shared_ptrapiHandler->areToolsAvailable(), std::move(ovmsCallback), streamerConfig); - } else { - legacyExecutionContext->textStreamer = std::make_shared( - getProperties()->tokenizer, - [& ctx = *legacyExecutionContext](std::string) -> ov::genai::StreamingStatus { - if (ctx.clientDisconnected.load()) { - return ov::genai::StreamingStatus::CANCEL; - } - return ov::genai::StreamingStatus::RUNNING; - }); } GenerationConfigBuilder configBuilder(getProperties()->baseGenerationConfig, getProperties()->toolParserName, @@ -203,10 +192,25 @@ absl::Status LegacyServable::prepareCompleteResponse(std::shared_ptrpayload.client->isDisconnected()) { return absl::CancelledError(); } - executionContext->response = executionContext->apiHandler->serializeUnaryResponse(legacyExecutionContext->results); - if (llm_calculator_logger->should_log(spdlog::level::debug)) { - logPerfMetrics(legacyExecutionContext->results.perf_metrics); + // By the time prepareCompleteResponse is called, readCompleteExecutionResults has + // already waited on finished — results and perf_metrics are fully populated. + executionContext->apiHandler->setPromptTokensUsage( + legacyExecutionContext->results.perf_metrics.get_num_input_tokens()); + executionContext->apiHandler->setCompletionTokensUsage( + legacyExecutionContext->results.perf_metrics.get_num_generated_tokens()); + + if (legacyExecutionContext->results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in legacy LLM unary generation result, defaulting to STOP"); + } + const ov::genai::GenerationFinishReason finishReason = + legacyExecutionContext->results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : legacyExecutionContext->results.finish_reasons[0]; + + if (executionContext->apiHandler->isVerboseResponse() && !legacyExecutionContext->results.tokens.empty()) { + executionContext->apiHandler->appendVerboseRawTokens(legacyExecutionContext->results.tokens[0]); } + + std::vector deltas = executionContext->deltaChannel.drain(); + executionContext->response = executionContext->apiHandler->serializeUnaryResponse(deltas, finishReason); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Complete unary response: {}", executionContext->response); return absl::OkStatus(); } @@ -216,91 +220,4 @@ absl::Status LegacyServable::readPartialExecutionResults(std::shared_ptr& executionContext) { - auto legacyExecutionContext = std::static_pointer_cast(executionContext); - if (legacyExecutionContext->payload.client->isDisconnected()) { - return absl::CancelledError(); - } - std::vector deltas = executionContext->deltaChannel.drain(); - const bool isFinishing = executionContext->deltaChannel.complete(); - if (!isFinishing) { - // For RESPONSES endpoint, always call serializeStreamingChunk so that - // output item initialization events are emitted even before the tokenizer produces text. - if (deltas.size() > 0 || executionContext->apiHandler->getEndpoint() == Endpoint::RESPONSES) { - for (auto& delta : deltas) { - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - std::move(delta), ov::genai::GenerationFinishReason::NONE); - if (!serialized.empty()) { - executionContext->response += wrapTextInServerSideEventMessage(serialized); - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Generated subsequent streaming response: {}", serialized); - } - } - if (deltas.empty()) { - // No delta generated yet — emit lifecycle events for RESPONSES endpoint. - if (!executionContext->lifecyclePrimed) { - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - rapidjson::Document{}, ov::genai::GenerationFinishReason::NONE); - if (!serialized.empty()) { - executionContext->response = wrapTextInServerSideEventMessage(serialized); - executionContext->lifecyclePrimed = true; - } - } - } - } - executionContext->sendLoopbackSignal = true; - } else { - // Wait for the readySignal - // (set right after pipe->generate() returns and results are assigned) - // to guarantee results is populated before we read finish_reasons and perf_metrics. - // Also ensures success flag is accurate. - legacyExecutionContext->finished.wait(); - if (!legacyExecutionContext->success) { - return absl::InvalidArgumentError("Request processing failed, check its correctness."); - } - OVMS_PROFILE_SCOPE("Generation of last streaming response"); - // end() was already called by pipe->generate() internally; all deltas are - // already in deltaChannel before signalComplete() fired. Drain any remaining. - for (auto& d : executionContext->deltaChannel.drain()) { - deltas.push_back(std::move(d)); - } - if (legacyExecutionContext->results.finish_reasons.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in legacy LM streaming generation result, defaulting to STOP"); - } - // Legacy generation path always runs with deltas=1, so we read the single finish reason at index 0. - ov::genai::GenerationFinishReason finishReason = legacyExecutionContext->results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : legacyExecutionContext->results.finish_reasons[0]; - if (executionContext->apiHandler->isVerboseResponse() && !legacyExecutionContext->results.tokens.empty()) { - executionContext->apiHandler->appendVerboseRawTokens(legacyExecutionContext->results.tokens[0]); - } - executionContext->apiHandler->setPromptTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_input_tokens()); - executionContext->apiHandler->setCompletionTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_generated_tokens()); - if (!deltas.empty()) { - for (size_t i = 0; i < deltas.size(); ++i) { - const bool isLast = (i == deltas.size() - 1); - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - std::move(deltas[i]), - isLast ? finishReason : ov::genai::GenerationFinishReason::NONE); - if (!serialized.empty()) { - executionContext->response += wrapTextInServerSideEventMessage(serialized); - } - } - } else { - // Parser produced no delta (generation ended on a swallowed token). - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - rapidjson::Document{}, finishReason); - if (!serialized.empty()) { - executionContext->response += wrapTextInServerSideEventMessage(serialized); - } - } - if (executionContext->apiHandler->getStreamOptions().includeUsage) - executionContext->response += wrapTextInServerSideEventMessage(executionContext->apiHandler->serializeStreamingUsageChunk()); - executionContext->response += wrapTextInServerSideEventMessage("[DONE]"); - if (llm_calculator_logger->should_log(spdlog::level::debug)) { - logPerfMetrics(legacyExecutionContext->results.perf_metrics); - } - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Generated complete streaming response: {}", executionContext->response); - executionContext->sendLoopbackSignal = false; - } - return absl::OkStatus(); -} - } // namespace ovms diff --git a/src/llm/language_model/legacy/servable.hpp b/src/llm/language_model/legacy/servable.hpp index f43b7693dd..99187ad250 100644 --- a/src/llm/language_model/legacy/servable.hpp +++ b/src/llm/language_model/legacy/servable.hpp @@ -26,13 +26,11 @@ namespace ovms { -struct LegacyServableExecutionContext : public GenAiServableExecutionContext { +struct LegacyServableExecutionContext : public LegacyServableExecutionContextBase { ov::genai::EncodedResults results; - std::promise readySignal; - std::future finished = readySignal.get_future(); + // readySignal, finished, success are inherited from LegacyServableExecutionContextBase // Workaround needed to pass generation config to the executor that requires it ov::genai::GenerationConfig baseGenerationConfig; - bool success{true}; // Disconnection handling std::atomic clientDisconnected{false}; @@ -41,6 +39,16 @@ struct LegacyServableExecutionContext : public GenAiServableExecutionContext { clientDisconnected = true; deltaChannel.signalComplete(); } + + // Legacy generation path always runs with a single beam, so finish_reasons[0] is the result. + ov::genai::GenerationFinishReason legacyFinishReason() const override { + return results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP + : results.finish_reasons[0]; + } + void setLegacyUsage(OpenAIApiHandler& apiHandler) override { + apiHandler.setPromptTokensUsage(results.perf_metrics.get_num_input_tokens()); + apiHandler.setCompletionTokensUsage(results.perf_metrics.get_num_generated_tokens()); + } }; struct LegacyServableProperties : public GenAiServableProperties { @@ -50,7 +58,7 @@ struct LegacyServableProperties : public GenAiServableProperties { int64_t maxPromptLength = 1024; // NPU property. 1024 is the default value in the plugin }; -class LegacyServable : public GenAiServable { +class LegacyServable : public LegacyServableBase { std::shared_ptr properties; void logPerfMetrics(ov::genai::PerfMetrics& perfMetrics); @@ -72,6 +80,5 @@ class LegacyServable : public GenAiServable { absl::Status readCompleteExecutionResults(std::shared_ptr& executionContext) override; absl::Status prepareCompleteResponse(std::shared_ptr& executionContext) override; absl::Status readPartialExecutionResults(std::shared_ptr& executionContext) override; - absl::Status preparePartialResponse(std::shared_ptr& executionContext) override; }; } // namespace ovms diff --git a/src/llm/omni_model/legacy/servable.cpp b/src/llm/omni_model/legacy/servable.cpp index dfb3838df7..3df1aaf513 100644 --- a/src/llm/omni_model/legacy/servable.cpp +++ b/src/llm/omni_model/legacy/servable.cpp @@ -111,13 +111,12 @@ absl::Status OmniModelLegacyServable::parseRequest(std::shared_ptrapiHandler->isStream()) { - if ((omniExecutionContext->apiHandler->getOutputParser() != nullptr && - omniExecutionContext->apiHandler->getOutputParser()->requiresStreamingWithSpecialTokens()) || - !omniExecutionContext->apiHandler->getRequest().skipSpecialTokens) { + const bool userWantsSpecialTokens = !omniExecutionContext->apiHandler->getRequest().skipSpecialTokens; + if (userWantsSpecialTokens) { streamerConfig.insert(ov::genai::skip_special_tokens(false)); } const bool audioRequested = omniExecutionContext->apiHandler->getRequest().audioOutputRequested; - auto ovmsCallback = [& ctx = *omniExecutionContext, audioRequested](rapidjson::Document delta, bool isLast) -> ov::genai::StreamingStatus { + auto ovmsCallback = [& ctx = *omniExecutionContext, audioRequested](Delta delta, bool isLast) -> ov::genai::StreamingStatus { if (ctx.clientDisconnected.load()) { ctx.deltaChannel.signalComplete(); return ov::genai::StreamingStatus::CANCEL; @@ -134,19 +133,15 @@ absl::Status OmniModelLegacyServable::parseRequest(std::shared_ptrapiHandler->getOutputParser() != nullptr && - omniExecutionContext->apiHandler->getOutputParser()->requiresStreamingWithSpecialTokens()) || - !omniExecutionContext->apiHandler->getRequest().skipSpecialTokens) { + const bool userWantsSpecialTokens = !omniExecutionContext->apiHandler->getRequest().skipSpecialTokens; + if (userWantsSpecialTokens) { streamerConfig.insert(ov::genai::skip_special_tokens(false)); } - auto unaryCallback = [& ctx = *omniExecutionContext](rapidjson::Document delta, bool /*isLast*/) -> ov::genai::StreamingStatus { + auto unaryCallback = [& ctx = *omniExecutionContext](Delta delta, bool isLast) -> ov::genai::StreamingStatus { if (ctx.clientDisconnected.load()) { return ov::genai::StreamingStatus::CANCEL; } - if (delta.HasMember("delta") && delta["delta"].IsObject() && - delta["delta"].HasMember("content") && delta["delta"]["content"].IsString()) { - ctx.accumulatedUnaryText += delta["delta"]["content"].GetString(); - } + ctx.deltaChannel.push(std::move(delta), isLast); return ov::genai::StreamingStatus::RUNNING; }; omniExecutionContext->textStreamer = std::make_shared( @@ -202,12 +197,7 @@ absl::Status OmniModelLegacyServable::parseRequest(std::shared_ptr(pcm16.data()), pcm16.size() * sizeof(int16_t))); - rapidjson::Document audioDoc; - audioDoc.SetObject(); - audioDoc.AddMember("_audio_delta", - rapidjson::Value(b64.c_str(), audioDoc.GetAllocator()), - audioDoc.GetAllocator()); - ctx.deltaChannel.push(std::move(audioDoc)); + ctx.deltaChannel.push(AudioDelta{std::move(b64)}); return ov::genai::StreamingStatus::RUNNING; }; } @@ -249,9 +239,13 @@ absl::Status OmniModelLegacyServable::prepareCompleteResponse(std::shared_ptraccumulatedUnaryText; + auto deltas = omniExecutionContext->deltaChannel.drain(); + const ov::genai::GenerationFinishReason finishReason = + omniExecutionContext->results.finish_reasons.empty() + ? ov::genai::GenerationFinishReason::STOP + : omniExecutionContext->results.finish_reasons[0]; executionContext->response = executionContext->apiHandler->serializeUnaryResponse( - omniExecutionContext->results, completeText); + deltas, finishReason); // If audio output was requested and waveforms are available, inject audio field into response if (omniExecutionContext->audioOutputRequested && !omniExecutionContext->results.speech_result.waveforms.empty()) { @@ -410,15 +404,15 @@ absl::Status OmniModelLegacyServable::preparePartialResponse(std::shared_ptrpayload.client->isDisconnected()) { return absl::CancelledError(); } - std::vector deltas = executionContext->deltaChannel.drain(); + std::vector deltas = executionContext->deltaChannel.drain(); const bool isFinishing = executionContext->deltaChannel.complete(); if (!isFinishing) { if (deltas.size() > 0 || executionContext->apiHandler->getEndpoint() == Endpoint::RESPONSES) { for (auto& delta : deltas) { - if (executionContext->apiHandler->isVerboseResponse() && - delta.HasMember("delta") && delta["delta"].IsObject() && - delta["delta"].HasMember("content") && delta["delta"]["content"].IsString()) { - executionContext->apiHandler->appendVerboseRawText(delta["delta"]["content"].GetString()); + if (executionContext->apiHandler->isVerboseResponse()) { + if (const auto* cd = std::get_if(&delta)) { + executionContext->apiHandler->appendVerboseRawText(cd->text); + } } std::string serialized = executionContext->apiHandler->serializeStreamingChunk( std::move(delta), ov::genai::GenerationFinishReason::NONE); @@ -430,7 +424,7 @@ absl::Status OmniModelLegacyServable::preparePartialResponse(std::shared_ptrlifecyclePrimed) { std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - rapidjson::Document{}, ov::genai::GenerationFinishReason::NONE); + FinishDelta{}, ov::genai::GenerationFinishReason::NONE); if (!serialized.empty()) { executionContext->response = wrapTextInServerSideEventMessage(serialized); executionContext->lifecyclePrimed = true; @@ -457,10 +451,10 @@ absl::Status OmniModelLegacyServable::preparePartialResponse(std::shared_ptrapiHandler->isVerboseResponse() && - deltas[i].HasMember("delta") && deltas[i]["delta"].IsObject() && - deltas[i]["delta"].HasMember("content") && deltas[i]["delta"]["content"].IsString()) { - executionContext->apiHandler->appendVerboseRawText(deltas[i]["delta"]["content"].GetString()); + if (executionContext->apiHandler->isVerboseResponse()) { + if (const auto* cd = std::get_if(&deltas[i])) { + executionContext->apiHandler->appendVerboseRawText(cd->text); + } } std::string serialized = executionContext->apiHandler->serializeStreamingChunk( std::move(deltas[i]), @@ -471,7 +465,7 @@ absl::Status OmniModelLegacyServable::preparePartialResponse(std::shared_ptrapiHandler->serializeStreamingChunk( - rapidjson::Document{}, finishReason); + FinishDelta{}, finishReason); if (!serialized.empty()) { executionContext->response += wrapTextInServerSideEventMessage(serialized); } diff --git a/src/llm/ovms_text_streamer.cpp b/src/llm/ovms_text_streamer.cpp index 02dd97300a..5764c6e290 100644 --- a/src/llm/ovms_text_streamer.cpp +++ b/src/llm/ovms_text_streamer.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include "../logging.hpp" namespace { // Matches GenAI's is_incomplete() in text_streamer.cpp. @@ -49,21 +49,102 @@ OVMSTextStreamer::OVMSTextStreamer( ov::genai::TextStreamer(tokenizer, noop_string_callback, decode_params), m_output_parser(output_parser), m_tools_available(tools_available), - m_callback(std::move(callback)) {} + m_callback(std::move(callback)) { + // Extract user's skip_special_tokens preference from decode_params. + // The OV any-map stores it as a bool under the canonical key name. + auto it = decode_params.find(ov::genai::skip_special_tokens.name()); + if (it != decode_params.end()) { + try { + // skip_special_tokens=true means we DON'T want special tokens. + const bool skip_special = it->second.as(); + m_user_wants_special = !skip_special; + } catch (...) { + } + } + // Sync m_additional_detokenization_params with the parser's initial phase requirements. + apply_decode_params(m_output_parser + ? m_output_parser->needSpecialTokensForCurrentDecode(m_user_wants_special) + : m_user_wants_special); +} + +void OVMSTextStreamer::apply_decode_params(bool decode_special_tokens) { + m_additional_detokenization_params[ov::genai::skip_special_tokens.name()] = !decode_special_tokens; + m_decode_special_tokens = decode_special_tokens; +} + +std::optional OVMSTextStreamer::handle_decoding_params_change(int64_t token) { + if (m_output_parser && !m_decode_special_tokens) { + const std::string startTag = m_output_parser->getPhaseStartTagForToken(token, m_tools_available); + if (!startTag.empty()) { + // Flush pending text with the current mode, then immediately flush the start + // tag so the phase switches before the next token's mode check fires. + if (!m_tokens_cache.empty()) { + const std::string pending = m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); + if (pending.size() > m_printed_len) { + const auto s = flush_chunk(pending, pending.size(), ov::genai::GenerationFinishReason::NONE); + if (s != ov::genai::StreamingStatus::RUNNING) + return s; + } + } + m_tokens_cache.clear(); + m_decoded_lengths.clear(); + m_printed_len = 0; + m_tokens_cache.push_back(token); + m_decoded_lengths.push_back(static_cast(startTag.size())); + const auto s = flush_chunk(startTag, startTag.size(), ov::genai::GenerationFinishReason::NONE); + m_tokens_cache.clear(); + m_decoded_lengths.clear(); + m_printed_len = 0; + apply_decode_params(m_output_parser->needSpecialTokensForCurrentDecode(m_user_wants_special)); + if (s != ov::genai::StreamingStatus::RUNNING) + return s; + return ov::genai::StreamingStatus::RUNNING; + } + } + + if (m_output_parser) { + const bool decode_with_special_tokens = m_output_parser->needSpecialTokensForCurrentDecode(m_user_wants_special); + if (decode_with_special_tokens != m_decode_special_tokens) { + if (!m_tokens_cache.empty()) { + const std::string text = m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); + if (text.size() > m_printed_len) { + const auto s = flush_chunk(text, text.size(), ov::genai::GenerationFinishReason::NONE); + if (s != ov::genai::StreamingStatus::RUNNING) + return s; + } + } + m_tokens_cache.clear(); + m_decoded_lengths.clear(); + m_printed_len = 0; + apply_decode_params(decode_with_special_tokens); + } + } + + return std::nullopt; +} -// ----------------------------------------------------------------------------- -// write(int64_t) — owned decode loop (does NOT delegate to TextStreamer::write) -// -// Replicates TextStreamer's flush heuristics: -// 1. Newline flush: emit immediately when text ends with '\n'. -// 2. Incomplete UTF-8 guard: if text ends with U+FFFD replacement char, mark as -1. -// 3. Delay buffer: hold back the last DELAY_N_TOKENS positions before flushing. -// -// Operates directly on the protected members inherited from TextStreamer: -// m_tokens_cache, m_decoded_lengths, m_printed_len, -// m_tokenizer, m_additional_detokenization_params. -// ----------------------------------------------------------------------------- ov::genai::StreamingStatus OVMSTextStreamer::write(int64_t token) { + if (llm_calculator_logger->should_log(spdlog::level::trace)) + m_all_tokens.push_back(token); + + return write(token, /*immediate_flush=*/false); +} + +ov::genai::StreamingStatus OVMSTextStreamer::write(const std::vector& tokens) { + ov::genai::StreamingStatus status = ov::genai::StreamingStatus::RUNNING; + for (const int64_t token : tokens) { + status = write(token); + if (status != ov::genai::StreamingStatus::RUNNING) { + return status; + } + } + return status; +} + +ov::genai::StreamingStatus OVMSTextStreamer::write(int64_t token, bool immediate_flush) { + if (const auto status = handle_decoding_params_change(token)) + return *status; + m_tokens_cache.push_back(token); const std::string text = m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); m_decoded_lengths.push_back(static_cast(text.size())); @@ -85,6 +166,14 @@ ov::genai::StreamingStatus OVMSTextStreamer::write(int64_t token) { return ov::genai::StreamingStatus::RUNNING; } + if (immediate_flush) { + // Flush this token's contribution immediately (no delay). + if (text.size() > m_printed_len) { + return flush_chunk(text, text.size(), ov::genai::GenerationFinishReason::NONE); + } + return ov::genai::StreamingStatus::RUNNING; + } + // 3. Delay buffer: need at least DELAY_N_TOKENS entries before flushing. const size_t n = m_decoded_lengths.size(); if (n < DELAY_N_TOKENS) { @@ -101,41 +190,61 @@ ov::genai::StreamingStatus OVMSTextStreamer::write(int64_t token) { ov::genai::GenerationFinishReason::NONE); } -ov::genai::StreamingStatus OVMSTextStreamer::write(const std::vector& tokens) { - ov::genai::StreamingStatus status = ov::genai::StreamingStatus::RUNNING; - for (const int64_t token : tokens) { - status = write(token); - if (status != ov::genai::StreamingStatus::RUNNING) { - return status; +void OVMSTextStreamer::end() { + if (llm_calculator_logger->should_log(spdlog::level::trace) && !m_all_tokens.empty()) { + const ov::AnyMap no_skip_params{{ov::genai::skip_special_tokens.name(), false}}; + const std::string full_decode = m_tokenizer.decode(m_all_tokens, no_skip_params); + std::string token_ids; + token_ids.reserve(m_all_tokens.size() * 7); + for (size_t i = 0; i < m_all_tokens.size(); ++i) { + if (i > 0) + token_ids += ", "; + token_ids += std::to_string(m_all_tokens[i]); } + SPDLOG_LOGGER_TRACE(llm_calculator_logger, + "OVMSTextStreamer: {} tokens generated; full decode (skip_special=false): \"{}\"; ids: [{}]", + m_all_tokens.size(), full_decode, token_ids); } - return status; -} -// ----------------------------------------------------------------------------- -// -// Decodes the remaining token cache (up to DELAY_N_TOKENS - 1 tokens that -// write() deliberately held back) and flushes with GenerationFinishReason::STOP. -// -// Does NOT call TextStreamer::end() — the base would fire its no-op callback -// and attempt to clear the protected state that we have already managed. -// ----------------------------------------------------------------------------- -void OVMSTextStreamer::end() { - // Always send a STOP flush so parsers that rely on finish_reason == STOP for - // cleanup (e.g. Hermes3 closing the argument string) receive the signal even - // when m_tokens_cache was cleared by a prior newline flush in write(). - if (!m_tokens_cache.empty()) { - const std::string text = m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); - if (text.size() > m_printed_len) { - flush_chunk(text, text.size(), ov::genai::GenerationFinishReason::STOP); + // Find the first token in m_tokens_cache that has not yet been printed. + // Tokens with decoded position <= m_printed_len were already flushed by write(); + // they must stay in the cache as BPE decode context for the tokens we drain below. + // A decoded length of -1 marks an incomplete UTF-8 sequence (also unprinted). + size_t first_unprinted_idx = 0; + while (first_unprinted_idx < m_decoded_lengths.size()) { + const int64_t dlen = m_decoded_lengths[first_unprinted_idx]; + if (dlen > 0 && static_cast(dlen) <= m_printed_len) { + first_unprinted_idx++; } else { - flush_chunk(text, m_printed_len, ov::genai::GenerationFinishReason::STOP); + break; } - } else { - // Cache already cleared (e.g. by a newline flush). No new text, but the - // STOP signal must still reach the parser. - flush_chunk("", 0, ov::genai::GenerationFinishReason::STOP); } + + // Extract the unprinted tokens; leave the printed ones in place as decode context. + const std::vector unprinted( + m_tokens_cache.begin() + static_cast(first_unprinted_idx), + m_tokens_cache.end()); + m_tokens_cache.resize(first_unprinted_idx); + m_decoded_lengths.resize(first_unprinted_idx); + // m_printed_len is intentionally kept as-is: it points to the end of the + // already-printed portion of the current cache so drainToken's flush starts + // at the right offset. + + for (const int64_t token : unprinted) { + const auto status = write(token, /*immediate_flush=*/true); + if (status != ov::genai::StreamingStatus::RUNNING) { + break; // cancelled mid-drain; still deliver the STOP signal below + } + } + + // Always deliver the STOP signal so parsers that rely on finishReason==STOP + // for cleanup receive it (e.g. hasPendingState flush in Lfm2ToolParser, + // argument string finalisation in Hermes3ToolParser). + const std::string final_text = m_tokens_cache.empty() + ? std::string{} + : m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); + flush_chunk(final_text, m_printed_len, ov::genai::GenerationFinishReason::STOP); + m_tokens_cache.clear(); m_decoded_lengths.clear(); m_printed_len = 0; @@ -149,8 +258,7 @@ void OVMSTextStreamer::end() { // last_idx = upper_bound(m_decoded_lengths, print_until) // // The resulting tokens sub-vector is passed to OutputParser::parseChunk alongside -// the decoded text chunk. All existing parsers ignore tokens in Phase 1; the -// parameter is available for future phase-aware parsers. +// the decoded text chunk. // // Callback is always fired when: // - parseChunk returns a non-nullopt Document, OR @@ -179,21 +287,11 @@ ov::genai::StreamingStatus OVMSTextStreamer::flush_chunk( m_printed_len = print_until; - std::optional delta; + std::optional delta; if (m_output_parser != nullptr) { delta = m_output_parser->parseChunk(chunk, tokens, m_tools_available, finish_reason); } else if (!chunk.empty()) { - // No parser: wrap raw text in a trivial {"delta":{"content":"..."}} document. - // Skip when chunk is empty (e.g. STOP flush after a newline-clearing write). - rapidjson::Document doc; - doc.SetObject(); - rapidjson::Document::AllocatorType& alloc = doc.GetAllocator(); - rapidjson::Value delta_obj(rapidjson::kObjectType); - delta_obj.AddMember("content", - rapidjson::Value(chunk.c_str(), alloc), - alloc); - doc.AddMember("delta", delta_obj, alloc); - delta = std::move(doc); + delta = ContentDelta{chunk}; } const bool isLast = (finish_reason != ov::genai::GenerationFinishReason::NONE); @@ -202,13 +300,9 @@ ov::genai::StreamingStatus OVMSTextStreamer::flush_chunk( } if (isLast) { // Parser produced no delta for the final flush (e.g. generation ended on a - // special token the parser absorbed). Still fire the callback with an empty - // object Document so the caller can emit the finish_reason chunk. - // Note: Document{} is kNullType; construct an empty object to avoid assertion - // failures in downstream code that calls HasMember() on the document. - rapidjson::Document empty; - empty.SetObject(); - return m_callback(std::move(empty), true); + // special token the parser absorbed). Fire the callback with FinishDelta so + // the caller can emit the finish_reason chunk. + return m_callback(FinishDelta{}, true); } return ov::genai::StreamingStatus::RUNNING; } diff --git a/src/llm/ovms_text_streamer.hpp b/src/llm/ovms_text_streamer.hpp index afc2fb7649..a41be270bb 100644 --- a/src/llm/ovms_text_streamer.hpp +++ b/src/llm/ovms_text_streamer.hpp @@ -22,47 +22,57 @@ #include #include -#include #include "io_processing/output_parser.hpp" namespace ovms { -// OVMSTextStreamer inherits ov::genai::TextStreamer to reuse its protected -// decode-loop state (m_tokenizer, m_tokens_cache, m_decoded_lengths, -// m_printed_len, m_additional_detokenization_params). It overrides -// write(int64_t) and end() completely — the no-op callback passed at -// construction is never invoked. +// OVMSTextStreamer is the bridge between the OpenVINO GenAI token generator and OutputParser. +// It is responsible for decoding raw token IDs into text and delivering correctly framed +// chunks to OutputParser::parseChunk(). Its behaviour is a precondition for OutputParser's +// correctness guarantees (see output_parser.hpp). // -// On every flush event the streamer: -// 1. Computes the token slice that produced the current text chunk via the -// same upper_bound logic used by ov::genai::TextParserStreamer. -// 2. Calls OutputParser::parseChunk(chunk, tokens, tools_available, finish_reason). -// 3. If the result is non-nullopt (or this is the final flush), fires the -// registered Callback with the Document. +// Guarantees provided by OVMSTextStreamer: +// - Ordered delivery: tokens are passed to OutputParser in the exact generation order, +// one logical chunk at a time. +// - Final flush: end() ALWAYS calls parseChunk("", [], finishReason=STOP) after all tokens +// have been processed. This is the "at least one subsequent call after every phase +// transition" guarantee that OutputParser depends on to drain buffered remainders. +// - Phase-aware decode mode: after every write(), the streamer queries +// OutputParser::needSpecialTokensForCurrentDecode() and adjusts skip_special_tokens for +// the next decode pass. This ensures structural special tokens (e.g. <|im_end|>) are +// visible as text during the phases whose parsers require them, and are suppressed +// (noise-free) in the content/unknown phase. +// - Token-ID phase detection: before decoding, write() checks +// OutputParser::getPhaseStartTagForToken() for the incoming token ID. If a match is +// found, the delay buffer is flushed immediately, the start-tag text is injected directly +// (without BPE decoding), and the decode mode is updated for the new phase — providing +// zero-latency phase entry without waiting for BPE to confirm the tag string. +// - Delay buffer: the last DELAY_N_TOKENS tokens are held back to prevent emitting partial +// BPE-fused text mid-word. Boundaries (phase starts, end()) force an immediate flush. // -// The Callback accumulates Documents in pendingDeltas on the execution context. -// preparePartialResponse drains pendingDeltas after each write()/end() cycle. +// OVMSTextStreamer does NOT perform any parsing, phase detection, or content routing. +// All of that is delegated to OutputParser. The streamer's sole responsibility is to +// ensure OutputParser receives correctly decoded, correctly ordered, and correctly framed +// input — including the mandatory final call on end(). // -// When output_parser is nullptr (e.g. /v1/completions endpoint), the streamer -// wraps the raw text in a trivial {"delta":{"content":"..."}} Document and -// fires the callback unconditionally, preserving existing behavior. +// Inherits ov::genai::TextStreamer to reuse its protected decode-loop state +// (m_tokenizer, m_tokens_cache, m_decoded_lengths, m_printed_len, +// m_additional_detokenization_params). write(int64_t) and end() are fully overridden; +// the no-op callback passed at construction is never invoked. class OVMSTextStreamer : public ov::genai::TextStreamer { public: - // Callback receives a Document and the isLast flag, and returns the streaming status. - // Document shape is always {"delta":{...}} matching the OpenAI delta format. // For the finish-only case (nullopt from parseChunk + STOP finishReason), - // an empty *object* Document (SetObject()) is passed so the caller can emit the finish_reason chunk. + // a FinishDelta{} is passed so the caller can emit the finish_reason chunk. // isLast is true when finish_reason != NONE — callers that push into a DeltaChannel - // should forward this flag to DeltaChannel::push() so the final document and the + // should forward this flag to DeltaChannel::push() so the final delta and the // completion signal are observed atomically (no separate signalComplete() needed). - using Callback = std::function; + using Callback = std::function; // outputParser may be nullptr (e.g. for the unary VLM path). - // TODO(phase3): rework ownership — OVMSTextStreamer should not need to keep - // the parser alive; it will be restructured in the next refactor phase. // toolsAvailable must be evaluated after parseRequest() has processed the body. - // decodeParams controls skip_special_tokens etc. — static for Phase 1. + // decodeParams controls skip_special_tokens etc. — the value is used as the baseline + // user preference; the parser's per-phase requirements are layered on top dynamically. OVMSTextStreamer( const ov::genai::Tokenizer& tokenizer, std::shared_ptr output_parser, @@ -71,28 +81,36 @@ class OVMSTextStreamer : public ov::genai::TextStreamer { const ov::AnyMap& decode_params); ov::genai::StreamingStatus write(int64_t token) override; - // TextStreamer::write(const vector&) calls ov::genai::TextStreamer::write(token) - // with a qualified (non-virtual) call, bypassing this class's write(int64_t) override. - // Override here to ensure our flush logic fires for every token. - // TODO(phase2): revisit once GenAI provides a cleaner extensibility hook. ov::genai::StreamingStatus write(const std::vector& tokens) override; void end() override; private: - // TODO(phase3): see constructor comment — ownership will be reworked. std::shared_ptr m_output_parser; bool m_tools_available; Callback m_callback; + // Whether the user's request specified skip_special_tokens=false. + bool m_user_wants_special = false; + // Whether the current decode pass should include special tokens (skip_special_tokens=false). + // Kept in sync with m_additional_detokenization_params via apply_decode_params. + bool m_decode_special_tokens = false; - // Must match the file-scope constexpr in openvino/genai text_streamer.cpp. - // Named here so a future GenAI change is a single update point. static constexpr size_t DELAY_N_TOKENS = 3; - // Flush text[m_printed_len : print_until] with the corresponding token slice. + void apply_decode_params(bool decode_special_tokens); + + // Flushes pending cache and switches decode mode; returns a status if the token was consumed as a phase-start token. + std::optional handle_decoding_params_change(int64_t token); + + // Like write() but with immediate_flush=true skips the delay buffer and flushes the token's contribution right away. + ov::genai::StreamingStatus write(int64_t token, bool immediate_flush); + ov::genai::StreamingStatus flush_chunk( const std::string& text, size_t print_until, ov::genai::GenerationFinishReason finish_reason); + + // All token IDs received by write() in order, used for end() trace logging. + std::vector m_all_tokens; }; } // namespace ovms diff --git a/src/llm/servable.cpp b/src/llm/servable.cpp index 638c536e37..cdf6787fbc 100644 --- a/src/llm/servable.cpp +++ b/src/llm/servable.cpp @@ -156,15 +156,13 @@ absl::Status GenAiServable::parseRequest(std::shared_ptrapiHandler->isStream()) { - auto ovmsCallback = [& ctx = *executionContext](rapidjson::Document delta, bool isLast) -> ov::genai::StreamingStatus { + { + auto ovmsCallback = [& ctx = *executionContext](Delta delta, bool isLast) -> ov::genai::StreamingStatus { ctx.deltaChannel.push(std::move(delta), isLast); return ov::genai::StreamingStatus::RUNNING; }; ov::AnyMap streamerConfig; - if ((executionContext->apiHandler->getOutputParser() != nullptr && - executionContext->apiHandler->getOutputParser()->requiresStreamingWithSpecialTokens()) || - !executionContext->apiHandler->getRequest().skipSpecialTokens) { + if (!executionContext->apiHandler->getRequest().skipSpecialTokens) { streamerConfig.insert(ov::genai::skip_special_tokens(false)); } executionContext->textStreamer = std::make_shared( @@ -257,7 +255,71 @@ absl::Status GenAiServable::prepareInputs(std::shared_ptr& executionContext) { - executionContext->response = executionContext->apiHandler->serializeUnaryResponse(executionContext->generationOutputs); + const bool hasLogprobs = executionContext->apiHandler->getRequest().logprobschat || + executionContext->apiHandler->getRequest().logprobs; + const size_t numOutputs = executionContext->generationOutputs.size(); + + // Build streamer config once; shared across all per-sequence streamers. + ov::AnyMap streamerConfig; + if (!executionContext->apiHandler->getRequest().skipSpecialTokens) { + streamerConfig.insert(ov::genai::skip_special_tokens(false)); + } + + std::vector> allDeltas; + std::vector finishReasons; + std::vector logprobData; + allDeltas.reserve(numOutputs); + finishReasons.reserve(numOutputs); + + for (size_t i = 0; i < numOutputs; ++i) { + const auto& output = executionContext->generationOutputs[i]; + + if (executionContext->apiHandler->isVerboseResponse()) { + executionContext->apiHandler->appendVerboseRawTokens(output.generated_ids); + } + executionContext->apiHandler->incrementProcessedTokens(output.generated_ids.size()); + + std::vector localDeltas; + if (numOutputs == 1) { + // Single sequence: reuse the OVMSTextStreamer and deltaChannel built in parseRequest. + executionContext->textStreamer->write(output.generated_ids); + executionContext->textStreamer->end(); + localDeltas = executionContext->deltaChannel.drain(); + } else { + // Multiple sequences: each beam requires its own independent stateful streamer + // (hold-back buffer, parser state are per-sequence). + auto cb = [&localDeltas](Delta delta, bool) -> ov::genai::StreamingStatus { + localDeltas.push_back(std::move(delta)); + return ov::genai::StreamingStatus::RUNNING; + }; + auto outputParser = executionContext->apiHandler->getOutputParser(); + if (outputParser) { + outputParser->resetStreamingState(); + } + auto tempStreamer = std::make_shared( + getProperties()->tokenizer, + outputParser, + executionContext->apiHandler->areToolsAvailable(), + std::move(cb), + streamerConfig); + tempStreamer->write(output.generated_ids); + tempStreamer->end(); + } + + allDeltas.push_back(std::move(localDeltas)); + finishReasons.push_back(output.finish_reason); + if (hasLogprobs) { + logprobData.push_back({output.generated_ids, output.generated_log_probs}); + } + } + + if (hasLogprobs) { + executionContext->response = executionContext->apiHandler->serializeUnaryResponse( + allDeltas, finishReasons, logprobData); + } else { + executionContext->response = executionContext->apiHandler->serializeUnaryResponse( + allDeltas, finishReasons); + } SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Complete unary response: {}", executionContext->response); return absl::OkStatus(); } @@ -292,7 +354,7 @@ absl::Status GenAiServable::preparePartialResponse(std::shared_ptr deltas = executionContext->deltaChannel.drain(); + std::vector deltas = executionContext->deltaChannel.drain(); const size_t count = deltas.size(); if (!isFinishing) { @@ -315,7 +377,7 @@ absl::Status GenAiServable::preparePartialResponse(std::shared_ptrlifecyclePrimed) { std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - rapidjson::Document{}, ov::genai::GenerationFinishReason::NONE); + FinishDelta{}, ov::genai::GenerationFinishReason::NONE); if (!serialized.empty()) { executionContext->response += wrapTextInServerSideEventMessage(serialized); executionContext->lifecyclePrimed = true; @@ -345,7 +407,7 @@ absl::Status GenAiServable::preparePartialResponse(std::shared_ptrapiHandler->serializeStreamingChunk( - rapidjson::Document{}, finishReason); + FinishDelta{}, finishReason); if (!serialized.empty()) { executionContext->response += wrapTextInServerSideEventMessage(serialized); } @@ -364,6 +426,102 @@ absl::Status GenAiServable::preparePartialResponse(std::shared_ptr& executionContext) { + auto legacyCtx = std::static_pointer_cast(executionContext); + if (legacyCtx->payload.client->isDisconnected()) { + return absl::CancelledError(); + } + std::vector deltas = executionContext->deltaChannel.drain(); + const bool isFinishing = executionContext->deltaChannel.complete(); + + // Helper: accumulate verbose raw text from a delta's content field. + // Both LLM-Legacy (switched from token-based) and VLM-Legacy use per-delta + // text extraction, which is correct because OVMSTextStreamer is configured with + // skip_special_tokens(false) in verbose mode, so delta content already includes + // special tokens. + auto appendVerboseContent = [&](const Delta& delta) { + if (executionContext->apiHandler->isVerboseResponse()) { + if (const auto* cd = std::get_if(&delta)) + executionContext->apiHandler->appendVerboseRawText(cd->text); + } + }; + + if (!isFinishing) { + // For RESPONSES endpoint, always call serializeStreamingChunk so that + // output item initialization events are emitted even before the tokenizer produces text. + if (deltas.size() > 0 || executionContext->apiHandler->getEndpoint() == Endpoint::RESPONSES) { + for (auto& delta : deltas) { + appendVerboseContent(delta); + std::string serialized = executionContext->apiHandler->serializeStreamingChunk( + std::move(delta), ov::genai::GenerationFinishReason::NONE); + if (!serialized.empty()) { + executionContext->response += wrapTextInServerSideEventMessage(serialized); + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Generated subsequent streaming response: {}", serialized); + } + } + if (deltas.empty()) { + // No delta generated yet — emit lifecycle events for RESPONSES endpoint. + if (!executionContext->lifecyclePrimed) { + std::string serialized = executionContext->apiHandler->serializeStreamingChunk( + FinishDelta{}, ov::genai::GenerationFinishReason::NONE); + if (!serialized.empty()) { + executionContext->response = wrapTextInServerSideEventMessage(serialized); + executionContext->lifecyclePrimed = true; + } + } + } + } + executionContext->sendLoopbackSignal = true; + } else { + // Wait for the readySignal + // (set right after pipe->generate() returns and results are assigned) + // to guarantee results is populated before we read finish_reasons and perf_metrics. + // Also ensures success flag is accurate. + legacyCtx->finished.wait(); + if (!legacyCtx->success) { + return absl::InvalidArgumentError("Request processing failed, check its correctness."); + } + OVMS_PROFILE_SCOPE("Generation of last streaming response"); + // end() was already called by pipe->generate() internally; all deltas are + // already in deltaChannel before signalComplete() fired. Drain any remaining. + for (auto& d : executionContext->deltaChannel.drain()) { + deltas.push_back(std::move(d)); + } + // Legacy generation path always runs with deltas=1, so we read the single finish reason at index 0. + const ov::genai::GenerationFinishReason finishReason = legacyCtx->legacyFinishReason(); + legacyCtx->setLegacyUsage(*executionContext->apiHandler); + if (!deltas.empty()) { + for (size_t i = 0; i < deltas.size(); ++i) { + const bool isLast = (i == deltas.size() - 1); + appendVerboseContent(deltas[i]); + std::string serialized = executionContext->apiHandler->serializeStreamingChunk( + std::move(deltas[i]), + isLast ? finishReason : ov::genai::GenerationFinishReason::NONE); + if (!serialized.empty()) { + executionContext->response += wrapTextInServerSideEventMessage(serialized); + } + } + } else { + // Parser produced no delta (generation ended on a swallowed token). + std::string serialized = executionContext->apiHandler->serializeStreamingChunk( + FinishDelta{}, finishReason); + if (!serialized.empty()) { + executionContext->response += wrapTextInServerSideEventMessage(serialized); + } + } + if (executionContext->apiHandler->getStreamOptions().includeUsage) + executionContext->response += wrapTextInServerSideEventMessage(executionContext->apiHandler->serializeStreamingUsageChunk()); + executionContext->response += wrapTextInServerSideEventMessage("[DONE]"); + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Generated complete streaming response: {}", executionContext->response); + executionContext->sendLoopbackSignal = false; + } + return absl::OkStatus(); +} + +absl::Status LegacyServableBase::preparePartialResponse(std::shared_ptr& executionContext) { + return prepareLegacyPartialResponse(executionContext); +} + void logRequestDetails(const ovms::HttpPayload& payload) { auto parsedJson = payload.parsedJson; rapidjson::StringBuffer buffer; diff --git a/src/llm/servable.hpp b/src/llm/servable.hpp index 4349bd249e..f8ee08f28c 100644 --- a/src/llm/servable.hpp +++ b/src/llm/servable.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -27,7 +28,7 @@ #pragma warning(disable : 4251 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246 6313) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include +#include "io_processing/delta.hpp" #include "openvino/genai/text_streamer.hpp" #include "mediapipe/framework/calculator_graph.h" #pragma GCC diagnostic pop @@ -88,7 +89,7 @@ struct DeltaChannel { // Push a delta from any thread (streamer callback). // When isLast is true, also marks the channel complete atomically so consumers // always see the final document and the completion flag in the same observation. - void push(rapidjson::Document delta, bool isLast = false) { + void push(Delta delta, bool isLast = false) { { std::lock_guard lock(m_mutex); m_deltas.push_back(std::move(delta)); @@ -117,9 +118,9 @@ struct DeltaChannel { } // Move all pending deltas out atomically. Returns an empty vector if none pending. - std::vector drain() { + std::vector drain() { std::lock_guard lock(m_mutex); - std::vector result; + std::vector result; result.swap(m_deltas); return result; } @@ -133,7 +134,7 @@ struct DeltaChannel { private: mutable std::mutex m_mutex; std::condition_variable m_cv; - std::vector m_deltas; + std::vector m_deltas; bool m_complete = false; }; @@ -154,6 +155,28 @@ struct GenAiServableExecutionContext { GenerationPhase generationPhase = GenerationPhase::INPUT_TOKEN_PROCESSING; }; +// Base execution context shared by all Legacy (non-CB) servables. +// Carries the synchronisation fields and a minimal type-erased interface that +// allows the shared preparePartialResponse implementation (prepareLegacyPartialResponse) +// to access type-specific results data without knowing the concrete results type. +struct LegacyServableExecutionContextBase : public GenAiServableExecutionContext { + std::promise readySignal; + std::future finished = readySignal.get_future(); + bool success{true}; + + // Returns the first finish reason from the concrete results, defaulting to STOP + // when the finish_reasons list is empty (e.g. cancelled or error path). + virtual ov::genai::GenerationFinishReason legacyFinishReason() const = 0; + // Forwards prompt and completion token counts from the concrete results into + // the handler's usage tracking fields. + virtual void setLegacyUsage(OpenAIApiHandler& apiHandler) = 0; + virtual ~LegacyServableExecutionContextBase() = default; +}; + +// Shared preparePartialResponse logic for both LLM-Legacy and VLM-Legacy servables. +// Defined in servable.cpp. Both Legacy servable overrides delegate here. +absl::Status prepareLegacyPartialResponse(std::shared_ptr& executionContext); + struct ExtraGenerationInfo { std::string bosTokenFromTokenizer; std::string bosTokenIdFromTokenizer; @@ -306,6 +329,21 @@ class GenAiServable { */ virtual absl::Status preparePartialResponse(std::shared_ptr& executionContext); }; + +// Intermediate base class for both LegacyServable and VisualLanguageModelLegacyServable. +// Provides the single shared override of preparePartialResponse that delegates to +// prepareLegacyPartialResponse, so neither concrete class needs to repeat it. +class LegacyServableBase : public GenAiServable { +public: + LegacyServableBase() = default; + LegacyServableBase(LegacyServableBase&&) = default; + LegacyServableBase& operator=(LegacyServableBase&&) = default; + LegacyServableBase(const LegacyServableBase&) = delete; + LegacyServableBase& operator=(const LegacyServableBase&) = delete; + + absl::Status preparePartialResponse(std::shared_ptr& executionContext) override; +}; + using GenAiServableMap = std::unordered_map>; void logRequestDetails(const HttpPayload& payload); } // namespace ovms diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index e0dcaddc08..054079c6f3 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -127,13 +127,11 @@ absl::Status VisualLanguageModelLegacyServable::parseRequest(std::shared_ptrapiHandler->isStream()) { - if ((legacyExecutionContext->apiHandler->getOutputParser() != nullptr && - legacyExecutionContext->apiHandler->getOutputParser()->requiresStreamingWithSpecialTokens()) || - !legacyExecutionContext->apiHandler->getRequest().skipSpecialTokens) { + { + if (!legacyExecutionContext->apiHandler->getRequest().skipSpecialTokens) { streamerConfig.insert(ov::genai::skip_special_tokens(false)); } - auto ovmsCallback = [& ctx = *legacyExecutionContext](rapidjson::Document delta, bool isLast) -> ov::genai::StreamingStatus { + auto ovmsCallback = [& ctx = *legacyExecutionContext](Delta delta, bool isLast) -> ov::genai::StreamingStatus { if (ctx.clientDisconnected.load()) { ctx.deltaChannel.signalComplete(); return ov::genai::StreamingStatus::CANCEL; @@ -147,41 +145,6 @@ absl::Status VisualLanguageModelLegacyServable::parseRequest(std::shared_ptrapiHandler->areToolsAvailable(), std::move(ovmsCallback), streamerConfig); - } else { - // For the unary path we still need OVMSTextStreamer so that the tokenizer - // decode params (e.g. skip_special_tokens) from the request are applied. - // results.texts[0] is decoded by the VLM pipeline with its own hardcoded - // config — using the streamer callback is the only way to respect the user's - // setting here. - // - // Crucially, we pass nullptr as the output parser: serializeUnaryResponse - // feeds accumulatedUnaryText back through encodeTextToTokens() and the - // batch parser (parseOutputIfNeeded), which expects raw decoded text with - // structural tags intact (e.g. , ). Passing a non-null - // parser here would strip those tags via parseChunk before accumulation - // and break the downstream unary parsing of reasoning/tool_calls. - // Will be further reworked in next refactor phases. - if ((legacyExecutionContext->apiHandler->getOutputParser() != nullptr && - legacyExecutionContext->apiHandler->getOutputParser()->requiresStreamingWithSpecialTokens()) || - !legacyExecutionContext->apiHandler->getRequest().skipSpecialTokens) { - streamerConfig.insert(ov::genai::skip_special_tokens(false)); - } - auto unaryCallback = [& ctx = *legacyExecutionContext](rapidjson::Document delta, bool /*isLast*/) -> ov::genai::StreamingStatus { - if (ctx.clientDisconnected.load()) { - return ov::genai::StreamingStatus::CANCEL; - } - if (delta.HasMember("delta") && delta["delta"].IsObject() && - delta["delta"].HasMember("content") && delta["delta"]["content"].IsString()) { - ctx.accumulatedUnaryText += delta["delta"]["content"].GetString(); - } - return ov::genai::StreamingStatus::RUNNING; - }; - legacyExecutionContext->textStreamer = std::make_shared( - getProperties()->tokenizer, - nullptr, // no parser: accumulate raw decoded text for batch unary parsing - false, - std::move(unaryCallback), - streamerConfig); } GenerationConfigBuilder configBuilder(getProperties()->baseGenerationConfig, getProperties()->toolParserName, @@ -229,14 +192,31 @@ absl::Status VisualLanguageModelLegacyServable::prepareCompleteResponse(std::sha return absl::CancelledError(); } - // pipe->generate() called streamer->end() before returning, so accumulatedUnaryText is - // already fully populated by the callbacks fired from OVMSTextStreamer::write()/end(). - const std::string& completeText = legacyExecutionContext->accumulatedUnaryText; - executionContext->response = executionContext->apiHandler->serializeUnaryResponse( - legacyExecutionContext->results, completeText); - if (llm_calculator_logger->should_log(spdlog::level::debug)) { - logPerfMetrics(legacyExecutionContext->results.perf_metrics); + // By the time prepareCompleteResponse is called, readCompleteExecutionResults has + // already waited on finished — results and perf_metrics are fully populated. + executionContext->apiHandler->setPromptTokensUsage( + legacyExecutionContext->results.perf_metrics.get_num_input_tokens()); + executionContext->apiHandler->setCompletionTokensUsage( + legacyExecutionContext->results.perf_metrics.get_num_generated_tokens()); + + if (legacyExecutionContext->results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in legacy VLM unary generation result, defaulting to STOP"); + } + const ov::genai::GenerationFinishReason finishReason = + legacyExecutionContext->results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : legacyExecutionContext->results.finish_reasons[0]; + + std::vector deltas = executionContext->deltaChannel.drain(); + + if (executionContext->apiHandler->isVerboseResponse()) { + for (const auto& delta : deltas) { + if (const auto* cd = std::get_if(&delta)) { + executionContext->apiHandler->appendVerboseRawText(cd->text); + } + } } + + executionContext->response = executionContext->apiHandler->serializeUnaryResponse( + deltas, finishReason); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Complete unary response: {}", executionContext->response); return absl::OkStatus(); } @@ -246,98 +226,4 @@ absl::Status VisualLanguageModelLegacyServable::readPartialExecutionResults(std: return absl::OkStatus(); } -absl::Status VisualLanguageModelLegacyServable::preparePartialResponse(std::shared_ptr& executionContext) { - auto legacyExecutionContext = std::static_pointer_cast(executionContext); - if (legacyExecutionContext->payload.client->isDisconnected()) { - return absl::CancelledError(); - } - std::vector deltas = executionContext->deltaChannel.drain(); - const bool isFinishing = executionContext->deltaChannel.complete(); - if (!isFinishing) { - // For RESPONSES endpoint, always call serializeStreamingChunk so that - // output item initialization events are emitted even before the tokenizer produces text. - if (deltas.size() > 0 || executionContext->apiHandler->getEndpoint() == Endpoint::RESPONSES) { - for (auto& delta : deltas) { - if (executionContext->apiHandler->isVerboseResponse() && - delta.HasMember("delta") && delta["delta"].IsObject() && - delta["delta"].HasMember("content") && delta["delta"]["content"].IsString()) { - executionContext->apiHandler->appendVerboseRawText(delta["delta"]["content"].GetString()); - } - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - std::move(delta), ov::genai::GenerationFinishReason::NONE); - if (!serialized.empty()) { - executionContext->response += wrapTextInServerSideEventMessage(serialized); - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Generated subsequent streaming response: {}", serialized); - } - } - if (deltas.empty()) { - // No delta generated yet — emit lifecycle events for RESPONSES endpoint. - if (!executionContext->lifecyclePrimed) { - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - rapidjson::Document{}, ov::genai::GenerationFinishReason::NONE); - if (!serialized.empty()) { - executionContext->response = wrapTextInServerSideEventMessage(serialized); - executionContext->lifecyclePrimed = true; - } - } - } - } - executionContext->sendLoopbackSignal = true; - } else { - // Wait for the readySignal - // (set right after pipe->generate() returns and results are assigned) - // to guarantee results is populated before we read finish_reasons and perf_metrics. - // Also ensures success flag is accurate. - legacyExecutionContext->finished.wait(); - if (!legacyExecutionContext->success) { - return absl::InvalidArgumentError("Request processing failed, check its correctness."); - } - OVMS_PROFILE_SCOPE("Generation of last streaming response"); - // end() was already called by pipe->generate() internally; all deltas are - // already in deltaChannel before signalComplete() fired. Drain any remaining. - for (auto& d : executionContext->deltaChannel.drain()) { - deltas.push_back(std::move(d)); - } - if (legacyExecutionContext->results.finish_reasons.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in legacy VLM streaming generation result, defaulting to STOP"); - } - // Legacy generation path always runs with deltas=1, so we read the single finish reason at index 0. - ov::genai::GenerationFinishReason finishReason = legacyExecutionContext->results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : legacyExecutionContext->results.finish_reasons[0]; - executionContext->apiHandler->setPromptTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_input_tokens()); - executionContext->apiHandler->setCompletionTokensUsage(legacyExecutionContext->results.perf_metrics.get_num_generated_tokens()); - if (!deltas.empty()) { - for (size_t i = 0; i < deltas.size(); ++i) { - const bool isLast = (i == deltas.size() - 1); - if (executionContext->apiHandler->isVerboseResponse() && - deltas[i].HasMember("delta") && deltas[i]["delta"].IsObject() && - deltas[i]["delta"].HasMember("content") && deltas[i]["delta"]["content"].IsString()) { - executionContext->apiHandler->appendVerboseRawText(deltas[i]["delta"]["content"].GetString()); - } - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - std::move(deltas[i]), - isLast ? finishReason : ov::genai::GenerationFinishReason::NONE); - if (!serialized.empty()) { - executionContext->response += wrapTextInServerSideEventMessage(serialized); - } - } - } else { - // Parser produced no delta (generation ended on a swallowed token). - std::string serialized = executionContext->apiHandler->serializeStreamingChunk( - rapidjson::Document{}, finishReason); - if (!serialized.empty()) { - executionContext->response += wrapTextInServerSideEventMessage(serialized); - } - } - if (executionContext->apiHandler->getStreamOptions().includeUsage) - executionContext->response += wrapTextInServerSideEventMessage(executionContext->apiHandler->serializeStreamingUsageChunk()); - executionContext->response += wrapTextInServerSideEventMessage("[DONE]"); - if (llm_calculator_logger->should_log(spdlog::level::debug)) { - logPerfMetrics(legacyExecutionContext->results.perf_metrics); - } - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Generated complete streaming response: {}", executionContext->response); - executionContext->sendLoopbackSignal = false; - } - return absl::OkStatus(); -} - } // namespace ovms diff --git a/src/llm/visual_language_model/legacy/servable.hpp b/src/llm/visual_language_model/legacy/servable.hpp index 3dfc253064..27f556ec2c 100644 --- a/src/llm/visual_language_model/legacy/servable.hpp +++ b/src/llm/visual_language_model/legacy/servable.hpp @@ -27,16 +27,11 @@ namespace ovms { -struct VisualLanguageModelLegacyServableExecutionContext : public GenAiServableExecutionContext { +struct VisualLanguageModelLegacyServableExecutionContext : public LegacyServableExecutionContextBase { ov::genai::VLMDecodedResults results; - std::promise readySignal; - std::future finished = readySignal.get_future(); + // readySignal, finished, success are inherited from LegacyServableExecutionContextBase // Workaround needed to pass generation config to the executor that requires it ov::genai::GenerationConfig baseGenerationConfig; - bool success{true}; - // Accumulated decoded text for the unary path — populated via OVMSTextStreamer - // callback so that the user's skip_special_tokens / decode params are respected. - std::string accumulatedUnaryText; // Disconnection handling std::atomic clientDisconnected{false}; @@ -45,6 +40,16 @@ struct VisualLanguageModelLegacyServableExecutionContext : public GenAiServableE clientDisconnected = true; deltaChannel.signalComplete(); } + + // Legacy generation path always runs with a single beam, so finish_reasons[0] is the result. + ov::genai::GenerationFinishReason legacyFinishReason() const override { + return results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP + : results.finish_reasons[0]; + } + void setLegacyUsage(OpenAIApiHandler& apiHandler) override { + apiHandler.setPromptTokensUsage(results.perf_metrics.get_num_input_tokens()); + apiHandler.setCompletionTokensUsage(results.perf_metrics.get_num_generated_tokens()); + } }; struct VisualLanguageModelLegacyServableProperties : public GenAiServableProperties { @@ -53,7 +58,7 @@ struct VisualLanguageModelLegacyServableProperties : public GenAiServablePropert std::shared_ptr legacyExecutor; }; -class VisualLanguageModelLegacyServable : public GenAiServable { +class VisualLanguageModelLegacyServable : public LegacyServableBase { std::shared_ptr properties; void logPerfMetrics(ov::genai::VLMPerfMetrics& perfMetrics); @@ -75,6 +80,5 @@ class VisualLanguageModelLegacyServable : public GenAiServable { absl::Status readCompleteExecutionResults(std::shared_ptr& executionContext) override; absl::Status prepareCompleteResponse(std::shared_ptr& executionContext) override; absl::Status readPartialExecutionResults(std::shared_ptr& executionContext) override; - absl::Status preparePartialResponse(std::shared_ptr& executionContext) override; }; } // namespace ovms diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index aaa82b1660..91a47900eb 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -550,22 +550,10 @@ TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensFalseNoPar EXPECT_EQ(apiHandler->getOutputParser(), nullptr); } -TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensTrueWithToolParser) { - std::string json = createRequestWithSkipSpecialTokensRawValue("true"); - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - std::optional maxTokensLimit; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - auto apiHandler = createHandler(endpoint(), "llama3"); - - EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - EXPECT_TRUE(apiHandler->getRequest().skipSpecialTokens); - EXPECT_NE(apiHandler->getOutputParser(), nullptr); -} - -TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensFalseWithToolParserSilentlyDisablesParser) { +// Requesting special tokens (skip_special_tokens=false) must only affect the CONTENT/UNKNOWN +// decode mode. A tool parser whose format needs no special tokens (llama3) must keep ignoring +// them once its own phase is active, regardless of what the caller asked for. +TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, UserSpecialTokensPreferenceOnlyAppliesInContentPhase) { std::string json = createRequestWithSkipSpecialTokensRawValue("false"); doc.Parse(json.c_str()); ASSERT_FALSE(doc.HasParseError()); @@ -574,40 +562,43 @@ TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensFalseWithT uint32_t bestOfLimit = 0; std::optional maxModelLength; auto apiHandler = createHandler(endpoint(), "llama3"); + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + auto outputParser = apiHandler->getOutputParser(); + ASSERT_NE(outputParser, nullptr); - EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - EXPECT_FALSE(apiHandler->getRequest().skipSpecialTokens); - EXPECT_EQ(apiHandler->getOutputParser(), nullptr); -} - -TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensFalseWithReasoningParserSilentlyDisablesParser) { - std::string json = createRequestWithSkipSpecialTokensRawValue("false"); - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); + // Still in UNKNOWN phase: the caller's preference is honoured here. + EXPECT_TRUE(outputParser->needSpecialTokensForCurrentDecode(/*userWantsSpecialTokens=*/true)); - std::optional maxTokensLimit; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - auto apiHandler = createHandler(endpoint(), "", "qwen3"); + // Drive the parser into the tool-call phase (llama3's start tag is consumed silently). + auto delta = outputParser->parseChunk("<|python_tag|>", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(delta.has_value()); - EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - EXPECT_FALSE(apiHandler->getRequest().skipSpecialTokens); - EXPECT_EQ(apiHandler->getOutputParser(), nullptr); + // llama3's tool body needs no special tokens: the phase must not inherit the user's preference. + EXPECT_FALSE(outputParser->needSpecialTokensForCurrentDecode(/*userWantsSpecialTokens=*/true)); } -TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensFalseWithBothParsersSilentlyDisablesParsers) { - std::string json = createRequestWithSkipSpecialTokensRawValue("false"); +// Symmetric case: a reasoning parser whose format DOES need special tokens (gemma4) must keep +// requiring them once its own phase is active, even when the caller asked for the opposite. +TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, ReasoningParserSpecialTokenNeedIgnoresUserPreference) { + std::string json = createRequestWithSkipSpecialTokensRawValue("true"); doc.Parse(json.c_str()); ASSERT_FALSE(doc.HasParseError()); std::optional maxTokensLimit; uint32_t bestOfLimit = 0; std::optional maxModelLength; - auto apiHandler = createHandler(endpoint(), "llama3", "qwen3"); + auto apiHandler = createHandler(endpoint(), "", "gemma4"); + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + auto outputParser = apiHandler->getOutputParser(); + ASSERT_NE(outputParser, nullptr); - EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - EXPECT_FALSE(apiHandler->getRequest().skipSpecialTokens); - EXPECT_EQ(apiHandler->getOutputParser(), nullptr); + // Drive the parser into the reasoning phase (gemma4's start tag is consumed silently). + auto delta = outputParser->parseChunk("<|channel>thought\n", {}, /*toolsAvailable=*/false, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(delta.has_value()); + + // gemma4 reasoning needs special tokens visible: still required even though the caller + // asked for skip_special_tokens=true (userWantsSpecialTokens=false). + EXPECT_TRUE(outputParser->needSpecialTokensForCurrentDecode(/*userWantsSpecialTokens=*/false)); } TEST_P(HttpOpenAIHandlerCommonParsingValidationTest, SkipSpecialTokensNotBoolFails) { @@ -923,36 +914,22 @@ INSTANTIATE_TEST_SUITE_P( } }); -static std::vector createHermes3ToolCallTokens(ov::genai::Tokenizer& tokenizer) { - std::string toolCall = R"({"name": "example_tool", "arguments": {"arg1": "value1", "arg2": 42}})"; - auto generatedTensor = tokenizer.encode(toolCall, ov::genai::add_special_tokens(true)).input_ids; - std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - return generatedTokens; -} - // Test helper: wraps the old serializeStreamingChunk(string, reason) behaviour for migration period. // Calls outputParser->parseChunk when a parser is present; otherwise builds a trivial content delta. static std::string serializeStreamingChunkFromText(ovms::OpenAIApiHandler& handler, const std::string& text, ov::genai::GenerationFinishReason finishReason) { const auto& outputParser = handler.getOutputParser(); - rapidjson::Document delta; if (outputParser != nullptr) { auto parsed = outputParser->parseChunk(text, {}, handler.areToolsAvailable(), finishReason); if (!parsed.has_value()) { if (finishReason == ov::genai::GenerationFinishReason::NONE) return ""; - delta = rapidjson::Document{}; - } else { - delta = std::move(*parsed); + return handler.serializeStreamingChunk(ovms::FinishDelta{}, finishReason); } - } else { - delta.SetObject(); - rapidjson::Document::AllocatorType& alloc = delta.GetAllocator(); - rapidjson::Value deltaObj(rapidjson::kObjectType); - deltaObj.AddMember("content", rapidjson::Value(text.c_str(), alloc), alloc); - delta.AddMember("delta", deltaObj, alloc); + return handler.serializeStreamingChunk(std::move(*parsed), finishReason); } + ovms::Delta delta = text.empty() ? ovms::Delta{ovms::FinishDelta{}} : ovms::Delta{ovms::ContentDelta{text}}; return handler.serializeStreamingChunk(std::move(delta), finishReason); } @@ -1084,15 +1061,34 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeStreamingChunkAlwaysIncludesDeltaF ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); // Simulate an empty Document (no "delta" member) with a finish reason — this - // is what flush_chunk produces when the parser absorbs the final token. - rapidjson::Document emptyDoc; - emptyDoc.SetObject(); - std::string serialized = apiHandler->serializeStreamingChunk(std::move(emptyDoc), ov::genai::GenerationFinishReason::LENGTH); + std::string serialized = apiHandler->serializeStreamingChunk(ovms::FinishDelta{}, ov::genai::GenerationFinishReason::LENGTH); ASSERT_NE(serialized.find("\"delta\":{}"), std::string::npos) << "Expected empty delta object in: " << serialized; ASSERT_NE(serialized.find("\"finish_reason\":\"length\""), std::string::npos) << serialized; } +// ---- serializeUnaryResponse(deltas, finishReason) tests ---- + +static ovms::ContentDelta makeContentDelta(const std::string& text) { + return ovms::ContentDelta{text}; +} + +static ovms::ReasoningDelta makeReasoningDelta(const std::string& text) { + return ovms::ReasoningDelta{text}; +} + +static ovms::ToolCallDelta makeToolCallFirstDelta(const std::string& id, const std::string& name, int index = 0) { + return ovms::ToolCallDelta{index, id, name, ""}; +} + +static ovms::ToolCallDelta makeToolCallArgsDelta(const std::string& args, int index = 0) { + return ovms::ToolCallDelta{index, std::nullopt, std::nullopt, args}; +} + +static ovms::FinishDelta makeFinishChunk() { + return ovms::FinishDelta{}; +} + TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseGenerationOutputReturnsToolCallsFinishReason) { std::string json = R"({ "model": "llama", @@ -1115,73 +1111,148 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseGenerationOutputRetur std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::GenerationOutput generationOutput; - generationOutput.generated_ids = createHermes3ToolCallTokens(*tokenizer); - generationOutput.finish_reason = ov::genai::GenerationFinishReason::STOP; // Change it once GenAI introduces tool_calls finish reason - std::string serialized = apiHandler->serializeUnaryResponse(std::vector{generationOutput}); + std::vector deltas; + deltas.push_back(makeToolCallFirstDelta("tc-001", "example_tool")); + deltas.push_back(makeToolCallArgsDelta("{\"arg1\":\"value1\"}")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"finish_reason\":\"tool_calls\""), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"tool_calls\":[{"), std::string::npos) << serialized; } -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseEncodedResultsReturnsToolCallsFinishReason) { +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseDeltasContentConcatenation) { + std::string json = R"({"model":"llama","messages":[{"role":"user","content":"Hi"}]})"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared( + doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + ASSERT_EQ(apiHandler->parseRequest(100, 0, std::nullopt), absl::OkStatus()); + + std::vector deltas; + deltas.push_back(makeContentDelta("Hello")); + deltas.push_back(makeContentDelta(", ")); + deltas.push_back(makeContentDelta("world!")); + deltas.push_back(makeFinishChunk()); + + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); + + ASSERT_NE(serialized.find("\"object\":\"chat.completion\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"finish_reason\":\"stop\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"content\":\"Hello, world!\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseIgnoresAudioDeltas) { + std::string json = R"({"model":"llama","messages":[{"role":"user","content":"Hi"}]})"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared( + doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + ASSERT_EQ(apiHandler->parseRequest(100, 0, std::nullopt), absl::OkStatus()); + + std::vector deltas; + deltas.push_back(makeContentDelta("Hello")); + deltas.push_back(ovms::AudioDelta{"aGVsbG8="}); + deltas.push_back(makeContentDelta(" world")); + deltas.push_back(makeFinishChunk()); + + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); + + ASSERT_NE(serialized.find("\"content\":\"Hello world\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"finish_reason\":\"stop\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseDeltasToolCallFragmentsAssembled) { std::string json = R"({ "model": "llama", - "stream": false, "messages": [{"role": "user", "content": "What is weather?"}], "tools": [{ "type": "function", - "function": { - "name": "example_tool", - "parameters": {"type": "object"} - } + "function": {"name": "get_weather", "parameters": {"type": "object"}} }] })"; doc.Parse(json.c_str()); ASSERT_FALSE(doc.HasParseError()); - auto apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer, "hermes3"); - uint32_t maxTokensLimit = 100; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + auto apiHandler = std::make_shared( + doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + ASSERT_EQ(apiHandler->parseRequest(100, 0, std::nullopt), absl::OkStatus()); + + std::vector deltas; + deltas.push_back(makeToolCallFirstDelta("tc-001", "get_weather", 0)); + deltas.push_back(makeToolCallArgsDelta("{\"loc\":", 0)); + deltas.push_back(makeToolCallArgsDelta("\"Paris\"}", 0)); + deltas.push_back(makeFinishChunk()); - ov::genai::EncodedResults results; - results.tokens = {createHermes3ToolCallTokens(*tokenizer)}; - std::string serialized = apiHandler->serializeUnaryResponse(results); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"finish_reason\":\"tool_calls\""), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"tool_calls\":[{"), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"name\":\"get_weather\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"arguments\":\"{\\\"loc\\\":\\\"Paris\\\"}\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"id\":\"tc-001\""), std::string::npos) << serialized; } -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseVLMSupportsToolCallsFinishReason) { - std::string json = R"({ - "model": "llama", - "stream": false, - "messages": [{"role": "user", "content": "What is weather?"}], - "tools": [{ - "type": "function", - "function": { - "name": "example_tool", - "parameters": {"type": "object"} - } - }] - })"; +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseDeltasReasoningContentPopulated) { + std::string json = R"({"model":"llama","messages":[{"role":"user","content":"Think"}]})"; doc.Parse(json.c_str()); ASSERT_FALSE(doc.HasParseError()); - auto apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer, "hermes3"); - uint32_t maxTokensLimit = 100; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + auto apiHandler = std::make_shared( + doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + ASSERT_EQ(apiHandler->parseRequest(100, 0, std::nullopt), absl::OkStatus()); - ov::genai::VLMDecodedResults results; - std::string toolCall = R"({"name": "example_tool", "arguments": {"arg1": "value1", "arg2": 42}})"; - results.texts = {toolCall}; - std::string serialized = apiHandler->serializeUnaryResponse(results, toolCall); + std::vector deltas; + deltas.push_back(makeReasoningDelta("Let me think...")); + deltas.push_back(makeReasoningDelta(" Done.")); + deltas.push_back(makeContentDelta("The answer is 42.")); + deltas.push_back(makeFinishChunk()); - ASSERT_NE(serialized.find("\"finish_reason\":\"tool_calls\""), std::string::npos) << serialized; + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); + + ASSERT_NE(serialized.find("\"reasoning_content\":\"Let me think... Done.\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"content\":\"The answer is 42.\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"finish_reason\":\"stop\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseDeltasLengthFinishReason) { + std::string json = R"({"model":"llama","messages":[{"role":"user","content":"Hi"}]})"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared( + doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + ASSERT_EQ(apiHandler->parseRequest(100, 0, std::nullopt), absl::OkStatus()); + + std::vector deltas; + deltas.push_back(makeContentDelta("Truncated")); + deltas.push_back(makeFinishChunk()); + + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::LENGTH); + + ASSERT_NE(serialized.find("\"finish_reason\":\"length\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseDeltasForResponsesHandler) { + std::string json = R"({"model":"llama","input":"What is OpenVINO?","max_output_tokens":5})"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared( + doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + ASSERT_EQ(apiHandler->parseRequest(std::nullopt, 0, std::nullopt), absl::OkStatus()); + + std::vector deltas; + deltas.push_back(makeContentDelta("OpenVINO is a toolkit.")); + deltas.push_back(makeFinishChunk()); + + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); + + ASSERT_NE(serialized.find("\"object\":\"response\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"type\":\"output_text\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("OpenVINO is a toolkit."), std::string::npos) << serialized; } TEST_F(HttpOpenAIHandlerParsingTest, ResponsesMultipleInputTextPartsPreservedAsContentArray) { @@ -1232,15 +1303,10 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesContainsO std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); + std::vector deltas; + deltas.push_back(makeContentDelta("OVMS")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"object\":\"response\""), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"output\":"), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"type\":\"output_text\""), std::string::npos) << serialized; @@ -1262,16 +1328,11 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesContainsR std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::EncodedResults results; - std::string modelOutput = "Let me reason about thisThe answer is 42"; - ov::Tensor outputIds = tokenizer->encode(modelOutput, ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); + std::vector deltas; + deltas.push_back(makeReasoningDelta("Let me reason about this")); + deltas.push_back(makeContentDelta("The answer is 42")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"object\":\"response\""), std::string::npos) << serialized; // Reasoning output item should be present ASSERT_NE(serialized.find("\"type\":\"reasoning\""), std::string::npos) << "Reasoning output item missing: " << serialized; @@ -1301,15 +1362,10 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesOmitsReas std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("OVMS is great", ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); + std::vector deltas; + deltas.push_back(makeContentDelta("OVMS is great")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"object\":\"response\""), std::string::npos) << serialized; // No reasoning output item when model output has no tags ASSERT_EQ(serialized.find("\"type\":\"reasoning\""), std::string::npos) << "Reasoning item should not be present: " << serialized; @@ -1344,10 +1400,11 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesOmitsEmpt std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::GenerationOutput generationOutput; - generationOutput.generated_ids = createHermes3ToolCallTokens(*tokenizer); - generationOutput.finish_reason = ov::genai::GenerationFinishReason::STOP; - std::string serialized = apiHandler->serializeUnaryResponse(std::vector{generationOutput}); + std::vector deltas; + deltas.push_back(makeToolCallFirstDelta("tc-001", "example_tool")); + deltas.push_back(makeToolCallArgsDelta("{}")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"object\":\"response\""), std::string::npos) << serialized; // The function_call output item must be present. @@ -1613,7 +1670,7 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeStreamingChunkEmptyPrimingDoesNotP // Empty priming call: should emit only lifecycle events, never output_text.delta, // and must not move the parser past the reasoning start tag. - std::string primingChunk = apiHandler->serializeStreamingChunk(rapidjson::Document{}, ov::genai::GenerationFinishReason::NONE); + std::string primingChunk = apiHandler->serializeStreamingChunk(ovms::FinishDelta{}, ov::genai::GenerationFinishReason::NONE); ASSERT_NE(primingChunk.find("\"type\":\"response.created\""), std::string::npos) << primingChunk; ASSERT_NE(primingChunk.find("\"type\":\"response.in_progress\""), std::string::npos) << primingChunk; ASSERT_EQ(primingChunk.find("\"type\":\"response.output_text.delta\""), std::string::npos) @@ -1890,17 +1947,10 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesIncomplet std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::GenerationOutput genOutput; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - genOutput.generated_ids = std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1]); - genOutput.finish_reason = ov::genai::GenerationFinishReason::LENGTH; - - std::vector generationOutputs = {genOutput}; - std::string serialized = apiHandler->serializeUnaryResponse(generationOutputs); + std::vector deltas; + deltas.push_back(makeContentDelta("OVMS")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::LENGTH); // Should have status "incomplete" ASSERT_NE(serialized.find("\"status\":\"incomplete\""), std::string::npos) << serialized; @@ -1933,17 +1983,10 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesCompleted std::optional maxModelLength; ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::GenerationOutput genOutput; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - genOutput.generated_ids = std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1]); - genOutput.finish_reason = ov::genai::GenerationFinishReason::STOP; - - std::vector generationOutputs = {genOutput}; - std::string serialized = apiHandler->serializeUnaryResponse(generationOutputs); + std::vector deltas; + deltas.push_back(makeContentDelta("OVMS")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); // Should have status "completed" ASSERT_NE(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; @@ -1958,173 +2001,6 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesCompleted ASSERT_NE(serialized.find("\"metadata\":{}"), std::string::npos) << serialized; } -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesEncodedResultsIncompleteOnLength) { - std::string json = R"({ - "model": "llama", - "input": "What is OpenVINO?", - "max_output_tokens": 5 - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); - std::optional maxTokensLimit; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - const auto& shape = outputIds.get_shape(); - ASSERT_EQ(shape.size(), 2); - ASSERT_EQ(shape[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + shape[1])}; - results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); - - ASSERT_NE(serialized.find("\"status\":\"incomplete\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"reason\":\"max_tokens\""), std::string::npos) << serialized; - ASSERT_EQ(serialized.find("\"completed_at\""), std::string::npos) << serialized; - ASSERT_EQ(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; -} - -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesEncodedResultsCompletedOnStop) { - std::string json = R"({ - "model": "llama", - "input": "What is OpenVINO?", - "max_output_tokens": 5 - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); - std::optional maxTokensLimit; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - results.finish_reasons = {ov::genai::GenerationFinishReason::STOP}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); - - ASSERT_NE(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"completed_at\""), std::string::npos) << serialized; - ASSERT_EQ(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; -} - -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesVLMDecodedResultsIncompleteOnLength) { - std::string json = R"({ - "model": "llama", - "input": "What is OpenVINO?", - "max_output_tokens": 5 - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); - std::optional maxTokensLimit; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - ov::genai::VLMDecodedResults results; - std::string text = "OVMS"; - results.texts = {text}; - results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; - - std::string serialized = apiHandler->serializeUnaryResponse(results, text); - - ASSERT_NE(serialized.find("\"status\":\"incomplete\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"reason\":\"max_tokens\""), std::string::npos) << serialized; - ASSERT_EQ(serialized.find("\"completed_at\""), std::string::npos) << serialized; - ASSERT_EQ(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; -} - -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesVLMDecodedResultsCompletedOnStop) { - std::string json = R"({ - "model": "llama", - "input": "What is OpenVINO?", - "max_output_tokens": 5 - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); - std::optional maxTokensLimit; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - ov::genai::VLMDecodedResults results; - std::string text = "OVMS"; - results.texts = {text}; - results.finish_reasons = {ov::genai::GenerationFinishReason::STOP}; - - std::string serialized = apiHandler->serializeUnaryResponse(results, text); - - ASSERT_NE(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"completed_at\""), std::string::npos) << serialized; - ASSERT_EQ(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; -} - -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseChatCompletionsEncodedResultsLengthFinishReason) { - std::string json = R"({ - "model": "llama", - "stream": false, - "messages": [{"role": "user", "content": "What is OpenVINO?"}] - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); - uint32_t maxTokensLimit = 100; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); - ASSERT_NE(serialized.find("\"finish_reason\":\"length\""), std::string::npos) << serialized; -} - -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseChatCompletionsVLMDecodedResultsLengthFinishReason) { - std::string json = R"({ - "model": "llama", - "stream": false, - "messages": [{"role": "user", "content": "What is OpenVINO?"}] - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); - uint32_t maxTokensLimit = 100; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - ov::genai::VLMDecodedResults results; - std::string text = "OVMS"; - results.texts = {text}; - results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; - - std::string serialized = apiHandler->serializeUnaryResponse(results, text); - ASSERT_NE(serialized.find("\"finish_reason\":\"length\""), std::string::npos) << serialized; -} - TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseCompletionsIncludesVerbosePayloadWhenEnabled) { std::string json = R"({ "model": "llama", @@ -2141,14 +2017,13 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseCompletionsIncludesVe ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); apiHandler->enableVerboseResponse("templated prompt"); + apiHandler->appendVerboseRawText("OVMS"); - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - + std::vector deltas; + deltas.push_back(makeContentDelta("OVMS")); + deltas.push_back(makeFinishChunk()); rapidjson::Document parsed; - parsed.Parse(apiHandler->serializeUnaryResponse(results).c_str()); + parsed.Parse(apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP).c_str()); ASSERT_FALSE(parsed.HasParseError()); ASSERT_TRUE(parsed.HasMember("__verbose")); ASSERT_TRUE(parsed["__verbose"].IsObject()); @@ -2172,46 +2047,13 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseCompletionsGeneration ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); apiHandler->enableVerboseResponse("templated prompt"); + apiHandler->appendVerboseRawText("OVMS"); - ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - ov::genai::GenerationOutput generationOutput; - generationOutput.generated_ids = std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1]); - generationOutput.finish_reason = ov::genai::GenerationFinishReason::STOP; - + std::vector deltas; + deltas.push_back(makeContentDelta("OVMS")); + deltas.push_back(makeFinishChunk()); rapidjson::Document parsed; - parsed.Parse(apiHandler->serializeUnaryResponse(std::vector{generationOutput}).c_str()); - ASSERT_FALSE(parsed.HasParseError()); - ASSERT_TRUE(parsed.HasMember("__verbose")); - ASSERT_TRUE(parsed["__verbose"].IsObject()); - ASSERT_STREQ(parsed["__verbose"]["prompt"].GetString(), "templated prompt"); - ASSERT_STREQ(parsed["__verbose"]["content"].GetString(), "OVMS"); -} - -TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseCompletionsVLMDecodedResultsIncludesVerbosePayloadWhenEnabled) { - std::string json = R"({ - "model": "llama", - "stream": false, - "prompt": "What is OpenVINO?" - })"; - doc.Parse(json.c_str()); - ASSERT_FALSE(doc.HasParseError()); - - auto apiHandler = std::make_shared(doc, ovms::Endpoint::COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); - uint32_t maxTokensLimit = 100; - uint32_t bestOfLimit = 0; - std::optional maxModelLength; - ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - - apiHandler->enableVerboseResponse("templated prompt"); - - ov::genai::VLMDecodedResults results; - std::string text = "OVMS"; - results.texts = {text}; - results.finish_reasons = {ov::genai::GenerationFinishReason::STOP}; - - rapidjson::Document parsed; - parsed.Parse(apiHandler->serializeUnaryResponse(results, text).c_str()); + parsed.Parse(apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP).c_str()); ASSERT_FALSE(parsed.HasParseError()); ASSERT_TRUE(parsed.HasMember("__verbose")); ASSERT_TRUE(parsed["__verbose"].IsObject()); @@ -2747,15 +2589,10 @@ TEST_F(HttpOpenAIHandlerParsingTest, SerializeResponsesUnaryResponseContainsFunc std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("Sunny", ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); + std::vector deltas; + deltas.push_back(makeContentDelta("Sunny")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"object\":\"response\""), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"tools\":[{"), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"type\":\"function\""), std::string::npos) << serialized; @@ -2792,15 +2629,10 @@ TEST_F(HttpOpenAIHandlerParsingTest, SerializeResponsesUnaryResponseContainsFunc std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::EncodedResults results; - ov::Tensor outputIds = tokenizer->encode("Sunny", ov::genai::add_special_tokens(false)).input_ids; - ASSERT_EQ(outputIds.get_shape().size(), 2); - ASSERT_EQ(outputIds.get_shape()[0], 1); - ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); - int64_t* outputIdsData = reinterpret_cast(outputIds.data()); - results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; - - std::string serialized = apiHandler->serializeUnaryResponse(results); + std::vector deltas; + deltas.push_back(makeContentDelta("Sunny")); + deltas.push_back(makeFinishChunk()); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); ASSERT_NE(serialized.find("\"tool_choice\":{"), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"type\":\"function\""), std::string::npos) << serialized; ASSERT_NE(serialized.find("\"name\":\"get_current_weather\""), std::string::npos) << serialized; @@ -3142,6 +2974,7 @@ TEST_F(HttpOpenAIHandlerParsingTest, OutputParserInitializationDependsOnParserNa auto withParserNames = std::make_shared( doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer, "llama3", ""); + ASSERT_EQ(withParserNames->parseRequest(/*maxTokensLimit=*/std::nullopt, /*bestOfLimit=*/0, /*maxModelLength=*/std::nullopt), absl::OkStatus()); EXPECT_NE(withParserNames->getOutputParser(), nullptr); } @@ -3184,12 +3017,13 @@ TEST_F(HttpOpenAIHandlerParsingTest, SerializeUnaryResponseVLMDecodedResultsWith ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); - ov::genai::VLMDecodedResults results; - std::string vlmText = - "I will call a tool.{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Paris\"}}"; - results.texts.push_back(vlmText); + std::vector deltas; + deltas.push_back(makeContentDelta("I will call a tool.")); + deltas.push_back(makeToolCallFirstDelta("tc-001", "get_weather")); + deltas.push_back(makeToolCallArgsDelta("{\"location\":\"Paris\"}")); + deltas.push_back(makeFinishChunk()); - std::string serialized = apiHandler->serializeUnaryResponse(results, vlmText); + std::string serialized = apiHandler->serializeUnaryResponse(deltas, ov::genai::GenerationFinishReason::STOP); rapidjson::Document responseDoc; responseDoc.Parse(serialized.c_str()); diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp index 6efc722efe..52d836a41c 100644 --- a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -27,6 +27,7 @@ #include "src/llm/io_processing/output_parser.hpp" #include "src/test/platform_utils.hpp" +#include "src/test/llm/output_parsers/output_parser_test_utils.hpp" using namespace ovms; @@ -100,7 +101,7 @@ class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { ToolsSchemas_t toolsSchemas = makeToolsSchemas(); OutputParser outputParser(tokenizer, "onyx", "onyx", toolsSchemas); - return outputParser.parse(generatedTokens, toolsAvailable); + return ovms::test::parseWithStreamer(tokenizer, outputParser, generatedTokens, toolsAvailable); } }; diff --git a/src/test/llm/io_processing_utils_test.cpp b/src/test/llm/io_processing_utils_test.cpp new file mode 100644 index 0000000000..c39ce6c917 --- /dev/null +++ b/src/test/llm/io_processing_utils_test.cpp @@ -0,0 +1,141 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include "src/llm/io_processing/utils.hpp" + +using namespace ovms; + +// ── basic hit / miss ───────────────────────────────────────────────────────── + +TEST(FindInStringTest, FindsTargetOutsideAllQuoting) { + EXPECT_EQ(findInStringRespectingSpecialChars("a, b, c", ",", 0), 1u); +} + +TEST(FindInStringTest, ReturnsNposWhenNotFound) { + EXPECT_EQ(findInStringRespectingSpecialChars("abc", ",", 0), std::string::npos); +} + +TEST(FindInStringTest, RespectsStartPos) { + EXPECT_EQ(findInStringRespectingSpecialChars("a, b, c", ",", 2), 4u); +} + +// ── double-quote depth ─────────────────────────────────────────────────────── + +TEST(FindInStringTest, SkipsTargetInsideDoubleQuotes) { + EXPECT_EQ(findInStringRespectingSpecialChars(R"("a, b", c)", ",", 0), 6u); +} + +TEST(FindInStringTest, FindsTargetAfterClosingDoubleQuote) { + EXPECT_EQ(findInStringRespectingSpecialChars(R"("abc", x)", ",", 0), 5u); +} + +TEST(FindInStringTest, EscapedDoubleQuoteDoesNotCloseQuoting) { + // \" inside a double-quoted string must not close it; comma is at 6. + EXPECT_EQ(findInStringRespectingSpecialChars(R"("a\"b", c)", ",", 0), 6u); +} + +TEST(FindInStringTest, SingleQuoteInsideDoubleQuoteIsIgnored) { + // Single quotes inside "..." must not affect singleQuoteDepth; comma at 6. + EXPECT_EQ(findInStringRespectingSpecialChars(R"("it's", b)", ",", 0), 6u); +} + +// ── brace / bracket depth ──────────────────────────────────────────────────── + +TEST(FindInStringTest, SkipsTargetInsideBraces) { + EXPECT_EQ(findInStringRespectingSpecialChars("{a, b}, c", ",", 0), 6u); +} + +TEST(FindInStringTest, SkipsTargetInsideBrackets) { + EXPECT_EQ(findInStringRespectingSpecialChars("[a, b], c", ",", 0), 6u); +} + +TEST(FindInStringTest, SkipsNestedDepth) { + // Comma inside inner [] still hidden; outer comma is the first one found. + EXPECT_EQ(findInStringRespectingSpecialChars("[a, [b, c], d], e", ",", 0), 14u); +} + +TEST(FindInStringTest, MixedBraceAndBracket) { + EXPECT_EQ(findInStringRespectingSpecialChars("{[a, b]}, c", ",", 0), 8u); +} + +// ── single-quote depth: opening ────────────────────────────────────────────── + +TEST(FindInStringTest, SingleQuoteAtStringStartOpens) { + // i == 0 → prevIsWord = false → quote opens; comma at 5. + EXPECT_EQ(findInStringRespectingSpecialChars("'abc', 'def'", ",", 0), 5u); +} + +TEST(FindInStringTest, SingleQuoteAfterNonWordOpens) { + // Non-word char before the quote → opens; comma at 6 (starting at 1). + EXPECT_EQ(findInStringRespectingSpecialChars("['abc', 'x']", ",", 1), 6u); +} + +TEST(FindInStringTest, ApostropheInWordDoesNotOpen) { + // Single quote between two word chars (it's) → not treated as opening quote. + EXPECT_EQ(findInStringRespectingSpecialChars("it's a test, b", ",", 0), 11u); +} + +// ── single-quote depth: closing ────────────────────────────────────────────── + +TEST(FindInStringTest, SingleQuoteClosedBeforeDelimiter) { + // Closing quote immediately followed by comma → singleQuoteDepth = 0. + std::string input = "['hello', 'world']"; + size_t pos = findInStringRespectingSpecialChars(input, ",", 1); + ASSERT_NE(pos, std::string::npos); + EXPECT_EQ(input[pos], ','); + EXPECT_EQ(pos, 8u); +} + +TEST(FindInStringTest, SingleQuoteClosedBeforeColon) { + // Closing quote followed by colon → closes; first comma is at 13. + EXPECT_EQ(findInStringRespectingSpecialChars("{'key': 'val', 'k2': 'v2'}", ",", 1), 13u); +} + +TEST(FindInStringTest, SingleQuoteClosedAtEndOfString) { + // j == str.size() branch: quote closes but target is absent → npos. + EXPECT_EQ(findInStringRespectingSpecialChars("'abc'", ",", 0), std::string::npos); +} + +TEST(FindInStringTest, SingleQuoteNotClosedBeforeNonDelimiter) { + // Closing quote followed by a regular letter → does NOT close; comma hidden. + EXPECT_EQ(findInStringRespectingSpecialChars("'abc' z, b", ",", 0), std::string::npos); +} + +TEST(FindInStringTest, ApostropheInWordNotClosingSingleQuote) { + // prevIsWord && nextIsWord → treated as plain char inside quoted string. + std::string input = "['it's the day', 'next']"; + size_t pos = findInStringRespectingSpecialChars(input, ",", 1); + EXPECT_NE(pos, std::string::npos); + EXPECT_GT(pos, 14u) << "comma found inside the single-quoted string"; +} + +TEST(FindInStringTest, PossessiveApostropheInWordNotClosingSingleQuote) { + // Word char before apostrophe, non-word non-delimiter after → does not close. + std::string input = "['Johns' car', 'other']"; + size_t pos = findInStringRespectingSpecialChars(input, ",", 1); + EXPECT_NE(pos, std::string::npos); + EXPECT_GT(pos, 12u) << "comma found inside the single-quoted string"; +} + +TEST(FindInStringTest, SingleQuoteClosedWithSpaceBeforeDelimiter) { + // j skips whitespace before checking for delimiter; quote still closes. + std::string input = "['abc' , 'def']"; + size_t pos = findInStringRespectingSpecialChars(input, ",", 1); + EXPECT_NE(pos, std::string::npos); + EXPECT_EQ(input[pos], ','); +} diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 12d77d98d3..0a013e72ee 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -2080,8 +2080,8 @@ TEST_P(LLMFlowHttpTestParameterized, inferChatCompletionsStream) { if (params.checkLogprobs) { ASSERT_FALSE(choice["logprobs"].IsObject()); } - if (choice.HasMember("delta")) { - ASSERT_TRUE(choice["delta"].IsObject()); + // "delta" may be an empty object {} in finish-reason-only chunks + if (choice["delta"].HasMember("content")) { ASSERT_TRUE(choice["delta"]["content"].IsString()); } } @@ -2131,7 +2131,10 @@ TEST_P(LLMFlowHttpTestParameterized, inferChatCompletionsStreamSkipSpecialTokens for (auto& choice : d["choices"].GetArray()) { if (choice.HasMember("delta")) { ASSERT_TRUE(choice["delta"].IsObject()); - ASSERT_TRUE(choice["delta"]["content"].IsString()); + // "delta" may be an empty object {} in finish-reason-only chunks + if (choice["delta"].HasMember("content")) { + ASSERT_TRUE(choice["delta"]["content"].IsString()); + } } } EXPECT_STREQ(d["object"].GetString(), "chat.completion.chunk"); diff --git a/src/test/llm/output_parsers/base_output_parser_test.cpp b/src/test/llm/output_parsers/base_output_parser_test.cpp index 031ff60b34..5dc3e4f149 100644 --- a/src/test/llm/output_parsers/base_output_parser_test.cpp +++ b/src/test/llm/output_parsers/base_output_parser_test.cpp @@ -15,67 +15,47 @@ //***************************************************************************** #include #include -#include "../../../llm/io_processing/base_output_parser.hpp" +#include "../../../llm/io_processing/delta.hpp" +#include "../../../llm/apis/openai_rapidjson_delta_serializer.hpp" +#include "src/port/rapidjson_document.hpp" using namespace ovms; -class BaseOutputParserTest : public ::testing::Test { -protected: - void SetUp() override { - // No specific setup needed for this test class - } -}; +class BaseOutputParserTest : public ::testing::Test {}; +// Verifies that ToolCallDelta{id, name} serializes to the OpenAI first-delta shape. TEST_F(BaseOutputParserTest, wrapFirstDelta) { - std::string functionName = "example_function"; - rapidjson::Document obj = BaseOutputParser::wrapFirstDelta(functionName, 0); - const auto& wrappedDelta = obj["delta"]; - ASSERT_TRUE(wrappedDelta.IsObject()); + std::string id = "abc123XYZ"; + std::string name = "example_function"; + ToolCallDelta d{0, id, name, ""}; + RapidJsonDeltaSerializer s; + std::string json = s.serialize(d); + + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + const auto& wrappedDelta = doc["delta"]; ASSERT_TRUE(wrappedDelta.HasMember("tool_calls")); - ASSERT_TRUE(wrappedDelta["tool_calls"].IsArray()); - ASSERT_EQ(wrappedDelta["tool_calls"].Size(), 1); - const auto& toolCall = wrappedDelta["tool_calls"][0]; - ASSERT_TRUE(toolCall.IsObject()); - ASSERT_TRUE(toolCall.HasMember("id")); - ASSERT_TRUE(toolCall["id"].IsString()); - std::string idStr = toolCall["id"].GetString(); - // Assuming ID is a random alphanumeric string of length 9 (see src/llm/io_processing/utils.cpp) - ASSERT_EQ(idStr.size(), 9); - ASSERT_TRUE(std::all_of(idStr.begin(), idStr.end(), [](char c) { - return std::isalnum(static_cast(c)); - })); - ASSERT_TRUE(toolCall.HasMember("type")); - ASSERT_EQ(toolCall["type"].GetString(), std::string("function")); - ASSERT_TRUE(toolCall.HasMember("index")); - ASSERT_EQ(toolCall["index"].GetInt(), 0); - ASSERT_TRUE(toolCall.HasMember("function")); - const auto& function = toolCall["function"]; - ASSERT_TRUE(function.IsObject()); - ASSERT_TRUE(function.HasMember("name")); - ASSERT_EQ(function["name"].GetString(), functionName); + const auto& tc = wrappedDelta["tool_calls"][0]; + ASSERT_EQ(std::string(tc["id"].GetString()), id); + ASSERT_EQ(std::string(tc["type"].GetString()), "function"); + ASSERT_EQ(tc["index"].GetInt(), 0); + ASSERT_EQ(std::string(tc["function"]["name"].GetString()), name); } +// Verifies that ToolCallDelta{nullopt, nullopt, args} serializes to the OpenAI args-delta shape. TEST_F(BaseOutputParserTest, wrapDelta) { - std::string deltaStr = R"({ - "arguments": "location" - })"; - rapidjson::Document delta; - delta.Parse(deltaStr.c_str()); + ToolCallDelta d{0, std::nullopt, std::nullopt, "location"}; + RapidJsonDeltaSerializer s; + std::string json = s.serialize(d); - rapidjson::Document obj = BaseOutputParser::wrapDelta(delta, 0); - const auto& wrappedDelta = obj["delta"]; - ASSERT_TRUE(wrappedDelta.IsObject()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + const auto& wrappedDelta = doc["delta"]; ASSERT_TRUE(wrappedDelta.HasMember("tool_calls")); - ASSERT_TRUE(wrappedDelta["tool_calls"].IsArray()); - ASSERT_EQ(wrappedDelta["tool_calls"].Size(), 1); - const auto& toolCall = wrappedDelta["tool_calls"][0]; - ASSERT_TRUE(toolCall.IsObject()); - ASSERT_TRUE(toolCall.HasMember("index")); - ASSERT_EQ(toolCall["index"].GetInt(), 0); - ASSERT_TRUE(toolCall.HasMember("function")); - const auto& function = toolCall["function"]; - ASSERT_TRUE(function.IsObject()); - ASSERT_TRUE(function.HasMember("arguments")); - ASSERT_TRUE(function["arguments"].IsString()); - ASSERT_EQ(function["arguments"].GetString(), std::string("location")); + const auto& tc = wrappedDelta["tool_calls"][0]; + ASSERT_EQ(tc["index"].GetInt(), 0); + ASSERT_FALSE(tc.HasMember("id")); + ASSERT_EQ(std::string(tc["function"]["arguments"].GetString()), "location"); } diff --git a/src/test/llm/output_parsers/default_content_parser_test.cpp b/src/test/llm/output_parsers/default_content_parser_test.cpp new file mode 100644 index 0000000000..a36aca7015 --- /dev/null +++ b/src/test/llm/output_parsers/default_content_parser_test.cpp @@ -0,0 +1,176 @@ +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include + +#include "src/llm/io_processing/default_content_parser.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +#ifdef _WIN32 +const std::string tokenizerPathDCP = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; +#else +const std::string tokenizerPathDCP = "/ovms/src/test/llm_testing/facebook/opt-125m"; +#endif + +class DefaultContentParserTest : public ::testing::Test { +protected: + static std::unique_ptr tokenizer; + + static void SetUpTestSuite() { + try { + tokenizer = std::make_unique(tokenizerPathDCP); + } catch (...) { + tokenizer = nullptr; + } + } + + static void TearDownTestSuite() { + tokenizer.reset(); + } + + static std::optional parseContent(DefaultContentParser& parser, const std::string& buf) { + auto result = parser.parseChunk(buf, {}, ov::genai::GenerationFinishReason::NONE); + if (!result.has_value()) + return std::nullopt; + const auto* cd = std::get_if(&*result); + EXPECT_NE(cd, nullptr) << "Expected ContentDelta"; + if (!cd) + return std::nullopt; + return cd->text; + } +}; + +std::unique_ptr DefaultContentParserTest::tokenizer; + +// ── No erase tags ────────────────────────────────────────────────────────── + +TEST_F(DefaultContentParserTest, PassThrough_NoTags) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer); + EXPECT_EQ(parseContent(parser, "hello world"), "hello world"); +} + +TEST_F(DefaultContentParserTest, EmptyBuffer_NoTags) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer); + EXPECT_EQ(parseContent(parser, ""), ""); +} + +// ── Single erase tag ─────────────────────────────────────────────────────── + +TEST_F(DefaultContentParserTest, TagFullyPresent_Erased) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|eot|>"}); + EXPECT_EQ(parseContent(parser, "<|eot|>"), ""); +} + +TEST_F(DefaultContentParserTest, TagInMiddle_SurroundingsKept) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|eom|>"}); + EXPECT_EQ(parseContent(parser, "before<|eom|>after"), "beforeafter"); +} + +TEST_F(DefaultContentParserTest, TagAtEnd_Erased) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|im_end|>"}); + EXPECT_EQ(parseContent(parser, "text<|im_end|>"), "text"); +} + +TEST_F(DefaultContentParserTest, TagAtStart_Erased) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {""}); + EXPECT_EQ(parseContent(parser, "text"), "text"); +} + +TEST_F(DefaultContentParserTest, TagRepeated_AllInstancesErased) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|eot|>"}); + EXPECT_EQ(parseContent(parser, "a<|eot|>b<|eot|>c"), "abc"); +} + +// ── Partial match → hold (return nullopt) ────────────────────────────────── + +TEST_F(DefaultContentParserTest, PartialTag_Hold) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|eot|>"}); + // Buffer ends with "<|eot" — suffix overlaps with prefix of "<|eot|>" + EXPECT_EQ(parseContent(parser, "text.<|eot"), std::nullopt); +} + +TEST_F(DefaultContentParserTest, SingleCharOverlap_Hold) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|im_end|>"}); + // Buffer ends with "<" which is the first char of "<|im_end|>" + EXPECT_EQ(parseContent(parser, "text<"), std::nullopt); +} + +// ── Multiple erase tags ──────────────────────────────────────────────────── + +TEST_F(DefaultContentParserTest, MultipleTags_AllErased) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"", "<|im_end|>"}); + EXPECT_EQ(parseContent(parser, "hello<|im_end|>"), "hello"); +} + +TEST_F(DefaultContentParserTest, MultipleTags_OnlyPresentOnesErased) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"", "<|im_end|>"}); + EXPECT_EQ(parseContent(parser, "hello"), "hello"); +} + +// ── FOUND_COMPLETE takes priority over FOUND_INCOMPLETE ─────────────────── + +TEST_F(DefaultContentParserTest, CompleteWinsOverIncomplete_Emits) { + if (!tokenizer) + GTEST_SKIP(); + // tag1 = "" is fully present; tag2 = "<|tool_response>" is partially overlapping. + // Expected: content is emitted (not held), with tag1 erased. + DefaultContentParser parser(*tokenizer, {"", "<|tool_response>"}); + // Buffer: "textend<" — "" FOUND_COMPLETE, "<" is first char of "<|tool_response>" FOUND_INCOMPLETE + auto result = parseContent(parser, "textend<"); + ASSERT_TRUE(result.has_value()) << "Should emit, not hold: FOUND_COMPLETE beats FOUND_INCOMPLETE"; + EXPECT_EQ(*result, "textend<"); +} + +// ── Content parser ignores tokens parameter ──────────────────────────────── + +TEST_F(DefaultContentParserTest, NonEmptyTokensIgnored) { + if (!tokenizer) + GTEST_SKIP(); + DefaultContentParser parser(*tokenizer, {"<|eot|>"}); + auto result = parser.parseChunk("hello", {1, 2, 3}, ov::genai::GenerationFinishReason::NONE); + ASSERT_TRUE(result.has_value()); + const auto* cd = std::get_if(&*result); + ASSERT_NE(cd, nullptr); + EXPECT_EQ(cd->text, "hello"); +} diff --git a/src/test/llm/output_parsers/devstral_output_parser_test.cpp b/src/test/llm/output_parsers/devstral_output_parser_test.cpp index a694e1e487..c087414376 100644 --- a/src/test/llm/output_parsers/devstral_output_parser_test.cpp +++ b/src/test/llm/output_parsers/devstral_output_parser_test.cpp @@ -20,6 +20,7 @@ #include "src/llm/io_processing/base_output_parser.hpp" #include "src/llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "test/platform_utils.hpp" using namespace ovms; @@ -85,7 +86,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithSingleToolCall) { std::string testInput = input; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -98,7 +99,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithSingleToolCall_MissingEn std::string testInput = "Reasoning before tool call [TOOL_CALLS] example_tool [ARGS]{\"arg1\":\"value1\",\"arg2\":42}"; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "Reasoning before tool call "); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -111,7 +112,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithSingleToolCall_EmptyArgu std::string testInput = "Reasoning before tool call [TOOL_CALLS]example_tool[ARGS]"; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "Reasoning before tool call "); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -124,7 +125,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = devstralTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -134,7 +135,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall std::string testInput = "Reasoning before tool call [TOOL_CALLS]example_tool[ARGS]{\"arg1\":\"value1\",\"arg2\":42}"; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "Reasoning before tool call "); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -147,8 +148,11 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithInvalidOrder) { std::string testInput = "Reasoning before tool call [ARGS]example_tool[TOOL_CALLS]{\"arg1\":\"value1\",\"arg2\":42}"; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, "Reasoning before tool call example_tool{\"arg1\":\"value1\",\"arg2\":42}"); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); + // [ARGS] appears before [TOOL_CALLS] (invalid order): [TOOL_CALLS] is consumed by OutputParser; + // [ARGS] is not a recognised start tag so it stays in content; chars after [TOOL_CALLS] are + // flushed as content when the devstral parser gives up waiting for [ARGS]. + EXPECT_EQ(parsedOutput.content, "Reasoning before tool call [ARGS]example_tool{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -158,7 +162,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithMissingArgsTag) { std::string testInput = input; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); // Same expected content as tokenizer does not add special tokens EXPECT_EQ(parsedOutput.content, "Some content example_tool{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -170,7 +174,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithArrayArguments) { std::string testInput = input; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -184,7 +188,7 @@ TEST_F(DevstralOutputParserTest, ParseToolCallOutputWithInvalidArguments) { std::string testInput = input; auto generatedTensor = devstralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*devstralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -225,15 +229,12 @@ TEST_F(DevstralOutputParserTest, HolisticStreaming) { int64_t chunkIteration = -1; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVecCopy) { chunkIteration++; - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -262,10 +263,7 @@ TEST_F(DevstralOutputParserTest, HolisticStreaming) { } else if (expectedDelta.has_value()) { FAIL() << "Mismatch for chunk: [" << chunk << "] got nothing but expected [" << expectedDelta.value() << "]" << chunkIteration; } else if (doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); FAIL() << "Mismatch for chunk: [" << chunk << "] expected nothing but got [" << docStr << "]" << chunkIteration; } else { FAIL() << "Mismatch for chunk: [" << chunk << "] " << chunkIteration; @@ -291,15 +289,12 @@ TEST_F(DevstralOutputParserTest, EmptyArgumentsStreaming) { int64_t chunkIteration = 0; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { chunkIteration++; - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -328,10 +323,7 @@ TEST_F(DevstralOutputParserTest, EmptyArgumentsStreaming) { } else if (expectedDelta.has_value()) { FAIL() << "Mismatch for chunk: [" << chunk << "] got nothing but expected [" << expectedDelta.value() << "]" << chunkIteration; } else if (doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); FAIL() << "Mismatch for chunk: [" << chunk << "] expected nothing but got [" << docStr << "]" << chunkIteration; } else { FAIL() << "Mismatch for chunk: [" << chunk << "] " << chunkIteration; @@ -354,15 +346,12 @@ TEST_F(DevstralOutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { diff --git a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp index 95b7df68a6..28612cdf00 100644 --- a/src/test/llm/output_parsers/gemma4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gemma4_output_parser_test.cpp @@ -24,6 +24,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -61,15 +62,12 @@ class Gemma4OutputParserTest : public ::testing::Test { outputParserWithRegularToolParsing = std::make_unique(*gemma4Tokenizer, "gemma4", "gemma4", EMPTY_TOOLS_SCHEMA); } - void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { if (!expectedDelta.has_value() && !doc.has_value()) { return; } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { @@ -79,15 +77,12 @@ class Gemma4OutputParserTest : public ::testing::Test { void assertStreamingVec(const std::vector>>& chunkToDeltaVec) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -116,10 +111,7 @@ class Gemma4OutputParserTest : public ::testing::Test { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -137,7 +129,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -155,7 +147,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithSingleToolCallAndReasoning for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, "Some reasoning content"); @@ -173,7 +165,7 @@ TEST_F(Gemma4OutputParserTest, ParseReasoningWithoutToolCall) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "SOME CONTENT WITHOUT TOOL CALL"); EXPECT_EQ(parsedOutput.reasoning, "Some reasoning content"); @@ -189,7 +181,9 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + // With no tools available this path should behave like plain text output, + // so Gemma control tokens are stripped during decode. + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, false); EXPECT_EQ(parsedOutput.content, inputWithoutSpecialTokens); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -204,7 +198,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithObjectArguments) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -222,7 +216,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArguments) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -240,7 +234,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -258,7 +252,7 @@ TEST_F(Gemma4OutputParserTest, ParserToolCallWithBooleanArgument) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -276,7 +270,7 @@ TEST_F(Gemma4OutputParserTest, ParseTwoToolCallsAtOnce) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -297,7 +291,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithArrayArguments) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -315,7 +309,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -353,7 +347,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentI for (auto& input : inputs) { auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "Before tool calls content. This is some content between tool calls. This is some content between second and third tool call. After tool calls content."); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -383,7 +377,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyArguments) { std::string input = "<|tool_call>call:no_args_tool{}"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{}"); @@ -394,7 +388,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithMultipleUtfChars) { std::string input = R"(<|tool_call>call:post_tweet{content:<|"|>Check out the sorted report! 🚀 We've made improvements to the content. Tagging @currenttech and mentioning Julia for our insightful team. #currenttech #trend<|"|>,mentions:[<|"|>@currenttech<|"|>,<|"|>Julia<|"|>],tags:[<|"|>#currenttrend<|"|>]})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "post_tweet"); @@ -445,7 +439,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -455,7 +449,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call>call:example_tool{arg1:<|\"|>value1<|\"|>,arg2:42}"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -633,7 +627,7 @@ TEST_F(Gemma4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { }; for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); assertChunkEqual(doc, expectedDelta, chunk); } } @@ -644,7 +638,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingParentheses) { std::string input = "<|tool_call>call:broken_tool"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -652,7 +646,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { std::string input = "<|tool_call>call:broken_tool{arg1:<|\"|>value1<|\"|>"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -660,7 +654,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithArgumentMissingEquals) { std::string input = "<|tool_call>call:broken{malformed_arg}"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); } @@ -669,7 +663,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingCompari std::string input = R"x(<|tool_call>call:search{query:<|"|>price >= 100, (sale)<|"|>,limit:5})x"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); @@ -680,7 +674,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesA std::string input = R"(<|tool_call>call:format{template:<|"|>Hello {name}, items: [a, b, c]<|"|>,count:3})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "format"); @@ -692,7 +686,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecial std::string input = R"(<|tool_call>call:execute{code:<|"|>)" + impl + R"(<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); @@ -703,7 +697,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingEscaped std::string input = R"x(<|tool_call>call:execute{code:<|"|>print(\"hello world\")<|"|>,verbose:true})x"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); @@ -714,7 +708,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingApostro std::string input = R"(<|tool_call>call:log{message:<|"|>it's a test, isn't it?<|"|>,level:<|"|>warn<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "log"); @@ -725,7 +719,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingBacksla std::string input = R"(<|tool_call>call:read_file{path:<|"|>C:\Users\test\file.txt<|"|>,encoding:<|"|>utf-8<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "read_file"); @@ -736,7 +730,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsC std::string input = R"(<|tool_call>call:save{lines:[<|"|>it's the wonderful day<|"|>,<|"|>He said: "My name's John"<|"|>,<|"|>That's Johns' car.<|"|>]})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -747,7 +741,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStrings std::string input = R"(<|tool_call>call:save{obj:{name:<|"|>it's the wonderful day<|"|>,greeting:<|"|>Hello, my name's Jan<|"|>,note:<|"|>That's Johns' car.<|"|>}})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -758,7 +752,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJ std::string input = R"(<|tool_call>call:send{payload:<|"|>{'key': 'value', 'count': 42}<|"|>,endpoint:<|"|>api<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "send"); @@ -769,7 +763,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithEmptyStringArgument) { std::string input = R"(<|tool_call>call:create{name:<|"|><|"|>,value:0})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "create"); @@ -780,7 +774,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { std::string input = R"(<|tool_call>call:translate{text:<|"|>zażółć gęślą jaźń<|"|>,lang:<|"|>pl<|"|>})"; auto generatedTensor = gemma4Tokenizer->encode(input).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); @@ -797,7 +791,7 @@ TEST_F(Gemma4OutputParserTest, ParseToolCallWithPythonCodeAsArgument) { print(f'\n\t{name} lives at {address}\n\r')<|"|>})x"; auto generatedTensor = gemma4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*gemma4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_tool"); diff --git a/src/test/llm/output_parsers/gptoss_output_parser_test.cpp b/src/test/llm/output_parsers/gptoss_output_parser_test.cpp index ea22c3be2e..8d9a13a0b4 100644 --- a/src/test/llm/output_parsers/gptoss_output_parser_test.cpp +++ b/src/test/llm/output_parsers/gptoss_output_parser_test.cpp @@ -21,6 +21,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" #include "../../../llm/io_processing/gptoss/harmony.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -465,15 +466,12 @@ class GptOssOutputStreamParserTest : public GptOssOutputUnaryParserTest { int64_t chunkIteration = -1; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVecCopy) { chunkIteration++; - std::optional doc = outputParser->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -502,10 +500,7 @@ class GptOssOutputStreamParserTest : public GptOssOutputUnaryParserTest { } else if (expectedDelta.has_value()) { FAIL() << "Mismatch for chunk: [" << chunk << "] got nothing but expected [" << expectedDelta.value() << "]" << chunkIteration; } else if (doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); FAIL() << "Mismatch for chunk: [" << chunk << "] expected nothing but got [" << docStr << "]" << chunkIteration; } else { FAIL() << "Mismatch for chunk: [" << chunk << "] " << chunkIteration; diff --git a/src/test/llm/output_parsers/hermes3_output_parser_test.cpp b/src/test/llm/output_parsers/hermes3_output_parser_test.cpp index 820d0fd19e..622ff33670 100644 --- a/src/test/llm/output_parsers/hermes3_output_parser_test.cpp +++ b/src/test/llm/output_parsers/hermes3_output_parser_test.cpp @@ -20,6 +20,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -68,13 +69,12 @@ TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithSingleToolCall) { for (auto& input : inputs) { auto generatedTensor = hermes3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*hermes3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -91,7 +91,7 @@ TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string testInput = input; auto generatedTensor = hermes3Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*hermes3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, false, true); EXPECT_EQ(parsedOutput.content, testInput); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -113,26 +113,23 @@ TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { for (auto& input : inputs) { auto generatedTensor = hermes3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*hermes3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 3); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated auto firstToolCallId = parsedOutput.toolCalls[0].id; EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated auto secondToolCallId = parsedOutput.toolCalls[1].id; EXPECT_NE(firstToolCallId, secondToolCallId); // IDs should be different EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); // ID should be generated auto thirdToolCallId = parsedOutput.toolCalls[2].id; @@ -141,46 +138,11 @@ TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { } } -TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithTwoValidToolCallsAndOneInvalid) { - std::string inputWithProperClosure = "{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}" - "{\"tool_name\": \"another_tool\", \"arguments\": {\"param1\": \"data\", \"param2\": true}}" - "{\"name\": \"third_tool\", \"arguments\": {\"key\": \"value\"}}"; - std::string inputWithImproperClosure = "{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}" - "{\"tool_name\": \"another_tool\", \"arguments\": {\"param1\": \"data\", \"param2\": true}}" - "{\"name\": \"third_tool\", \"arguments\": {\"key\": \"value\"}}"; - - // Hermes3 may produce last tool call without closing tag, so we test both cases - // The results should be identical - std::vector inputs = {inputWithProperClosure, inputWithImproperClosure}; - for (auto& input : inputs) { - auto generatedTensor = hermes3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; - std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, ""); - EXPECT_EQ(parsedOutput.reasoning, ""); - - // Expecting two tool calls as the second one does not have a valid name - ASSERT_EQ(parsedOutput.toolCalls.size(), 2); - EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated - auto firstToolCallId = parsedOutput.toolCalls[0].id; - - EXPECT_EQ(parsedOutput.toolCalls[1].name, "third_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces - EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"key\":\"value\"}"); - EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated - auto secondToolCallId = parsedOutput.toolCalls[1].id; - EXPECT_NE(firstToolCallId, secondToolCallId); // IDs should be different - } -} - TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = hermes3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*hermes3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -191,13 +153,12 @@ TEST_F(Hermes3OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) auto generatedTensor = hermes3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); // generatedTokens should now contain content followed by bot token ID and then tool call - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*hermes3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -339,15 +300,12 @@ TEST_F(Hermes3OutputParserTest, HolisticStreaming) { }; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -376,10 +334,7 @@ TEST_F(Hermes3OutputParserTest, HolisticStreaming) { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -419,15 +374,12 @@ TEST_F(Hermes3OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { diff --git a/src/test/llm/output_parsers/lfm25_output_parser_test.cpp b/src/test/llm/output_parsers/lfm25_output_parser_test.cpp index fbb6062cad..35af9cc2d6 100644 --- a/src/test/llm/output_parsers/lfm25_output_parser_test.cpp +++ b/src/test/llm/output_parsers/lfm25_output_parser_test.cpp @@ -24,6 +24,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -61,15 +62,12 @@ class LFM25OutputParserTest : public ::testing::Test { outputParserWithRegularToolParsing = std::make_unique(*lfm25Tokenizer, "lfm2", "lfm2", EMPTY_TOOLS_SCHEMA); } - void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { if (!expectedDelta.has_value() && !doc.has_value()) { return; } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { @@ -91,15 +89,12 @@ class LFM25OutputParserTest : public ::testing::Test { void assertStreamingVec(const std::vector>>& chunkToDeltaVec) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { auto tokens = encodeChunk(chunk); - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, tokens, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, tokens, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -128,10 +123,7 @@ class LFM25OutputParserTest : public ::testing::Test { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -149,7 +141,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithSingleToolCall) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -161,6 +153,21 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithSingleToolCall) { } } +TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithSingleToolCall_ProductionMode) { + // Verifies the proactive token-ID phase-start switch works when the user has not + // requested special tokens; without it <|tool_call_start|> decodes to empty string. + std::string input = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; + auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, false); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string inputWithProperClosure = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; @@ -169,7 +176,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string testInput = input; auto generatedTensor = lfm25Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, false, true); EXPECT_EQ(parsedOutput.content, testInput); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -184,7 +191,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithObjectArguments) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -203,7 +210,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArguments) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -222,7 +229,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -241,7 +248,7 @@ TEST_F(LFM25OutputParserTest, ParserToolCallWithBooleanArgument) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -260,7 +267,7 @@ TEST_F(LFM25OutputParserTest, ParseTwoToolCallsAtOnce) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -282,7 +289,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithArrayArguments) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -301,7 +308,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -322,7 +329,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -360,7 +367,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentIn for (auto& input : inputs) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "Before tool calls content. This is some content between tool calls. This is some content between second and third tool call. After tool calls content."); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -390,7 +397,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithEmptyArguments) { std::string input = "<|tool_call_start|>[no_args_tool()]<|tool_call_end|>"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); } @@ -399,7 +406,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -409,7 +416,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -748,7 +755,7 @@ TEST_F(LFM25OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); assertChunkEqual(doc, expectedDelta, chunk); } } @@ -759,7 +766,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithMissingParentheses) { std::string input = "<|tool_call_start|>[broken_tool]<|tool_call_end|>"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -767,7 +774,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { std::string input = "<|tool_call_start|>[broken_tool(arg1=\"value1\"]<|tool_call_end|>"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -776,7 +783,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithArgumentMissingEquals) { std::string input = "<|tool_call_start|>[broken(malformed_arg)]<|tool_call_end|>"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); // The tool call is parsed but the argument value will be empty and invalid ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); @@ -786,7 +793,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithMissingSquareBracket) { std::string input = "<|tool_call_start|>broken(arg1=1)<|tool_call_end|>"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -796,7 +803,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingComparis std::string input = R"x(<|tool_call_start|>[search(query="price >= 100, (sale)", limit=5)]<|tool_call_end|>)x"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); @@ -807,7 +814,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesAn std::string input = R"(<|tool_call_start|>[format(template="Hello {name}, items: [a, b, c]", count=3)]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "format"); @@ -819,7 +826,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecialC std::string input = R"(<|tool_call_start|>[execute(code=")" + impl + R"(")]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); @@ -830,7 +837,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingEscapedQ std::string input = R"x(<|tool_call_start|>[execute(code="print(\"hello world\")", verbose=true)]<|tool_call_end|>)x"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); @@ -841,7 +848,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingApostrop std::string input = R"(<|tool_call_start|>[log(message="it's a test, isn't it?", level="warn")]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "log"); @@ -852,7 +859,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingBackslas std::string input = R"(<|tool_call_start|>[read_file(path="C:\Users\test\file.txt", encoding="utf-8")]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "read_file"); @@ -863,7 +870,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsCo std::string input = R"(<|tool_call_start|>[save(lines=['it's the wonderful day', 'My name's Jan', 'That's Johns' car.'])]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -874,7 +881,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentContainingSingleQuo std::string input = R"(<|tool_call_start|>[save(line="I've had line with single quotes")]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -885,7 +892,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsC std::string input = R"(<|tool_call_start|>[save(obj={'name':'it's the wonderful day', 'greeting':'Hello, my name's Jan', 'note':'That's Johns' car.'})]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -896,7 +903,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJS std::string input = R"(<|tool_call_start|>[send(payload="{'key': 'value', 'count': 42}", endpoint="api")]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "send"); @@ -907,7 +914,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithEmptyStringArgument) { std::string input = R"(<|tool_call_start|>[create(name="", value=0)]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "create"); @@ -918,7 +925,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { std::string input = R"(<|tool_call_start|>[translate(text="zażółć gęślą jaźń", lang="pl")]<|tool_call_end|>)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); @@ -935,7 +942,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithPythonCodeAsArgument) { print(f'\n\t{name} lives at {address}\n\r')")]<|tool_call_end|>)x"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_tool"); @@ -947,7 +954,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithReasoning) { auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); @@ -959,7 +966,7 @@ TEST_F(LFM25OutputParserTest, ParseToolCallWithReasoningAndContent) { std::string input = R"(User wants me to translate string "zażółć gęślą jaźń" from polish. Polish parameter language signature is "pl". I should use function translate. [...]<|tool_call_start|>[translate(text="zażółć gęślą jaźń", lang="pl")]<|tool_call_end|> This is the content after the tool call.)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, " This is the content after the tool call."); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); @@ -971,7 +978,7 @@ TEST_F(LFM25OutputParserTest, ParseOutputWithReasoningAndContent) { std::string input = R"(User wants me to answer what is the difference between "foo" and "bar". I should answer with a short explanation. [...] The difference between "foo" and "bar" is that "foo" is often used as a placeholder name in programming, while "bar" is another placeholder name that is commonly used alongside "foo".)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, " The difference between \"foo\" and \"bar\" is that \"foo\" is often used as a placeholder name in programming, while \"bar\" is another placeholder name that is commonly used alongside \"foo\"."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, R"(User wants me to answer what is the difference between "foo" and "bar". I should answer with a short explanation. [...])"); @@ -981,7 +988,7 @@ TEST_F(LFM25OutputParserTest, ParseOutputWithoutReasoningAndTools) { std::string input = R"(This is a simple output without reasoning and tools.)"; auto generatedTensor = lfm25Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm25Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a simple output without reasoning and tools."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); diff --git a/src/test/llm/output_parsers/lfm2_output_parser_test.cpp b/src/test/llm/output_parsers/lfm2_output_parser_test.cpp index ff5bdf62b7..96663ff147 100644 --- a/src/test/llm/output_parsers/lfm2_output_parser_test.cpp +++ b/src/test/llm/output_parsers/lfm2_output_parser_test.cpp @@ -24,6 +24,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -61,15 +62,12 @@ class LFM2OutputParserTest : public ::testing::Test { outputParserWithRegularToolParsing = std::make_unique(*lfm2Tokenizer, "lfm2", "", EMPTY_TOOLS_SCHEMA); } - void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + void assertChunkEqual(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { if (!expectedDelta.has_value() && !doc.has_value()) { return; } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { @@ -78,15 +76,12 @@ class LFM2OutputParserTest : public ::testing::Test { } void assertStreamingVec(const std::vector>>& chunkToDeltaVec) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -115,10 +110,7 @@ class LFM2OutputParserTest : public ::testing::Test { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -136,18 +128,53 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithSingleToolCall) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } } +TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithSingleToolCall_ProductionMode) { + // Production mode: userWantsSpecialTokens=false (skip_special_tokens=true by default). + // The <|tool_call_start|> and <|tool_call_end|> tokens are special — without the + // proactive isPhaseStartToken() switch in OVMSTextStreamer::write they would decode + // to empty strings and tool-call detection would silently fail. + std::string input = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; + auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, false); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithSingleToolCall_ToolOnlyProductionMode) { + // Tool-only parser (no reasoning parser), production mode (userWantsSpecialTokens=false). + // This is the configuration that actually exercises the isPhaseStartToken() proactive + // flush in OVMSTextStreamer::write(): with no reasoning parser present, there is no + // defaultDecodingWithSpecialTokens source to keep the mode on, so <|tool_call_start|> + // would decode to empty text without the proactive switch, silently losing all tool calls. + auto toolOnlyParser = std::make_unique(*lfm2Tokenizer, "lfm2", "", EMPTY_TOOLS_SCHEMA); + std::string input = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; + auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *toolOnlyParser, generatedTokens, true, false); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string inputWithProperClosure = "<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; @@ -156,7 +183,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithNoToolsInTheRequest) { std::string testInput = input; auto generatedTensor = lfm2Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, false, true); EXPECT_EQ(parsedOutput.content, testInput); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -171,13 +198,12 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithObjectArguments) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -190,13 +216,12 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArguments) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "test1"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"data1, data2\"}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -209,13 +234,12 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithListOfStringsAsArgument) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "generate_DNA_sequence"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"length\":100,\"preferences\":[\"G\",\"C\"]}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -228,13 +252,12 @@ TEST_F(LFM2OutputParserTest, ParserToolCallWithBooleanArgument) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "check_status"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"flag\":true}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -247,14 +270,13 @@ TEST_F(LFM2OutputParserTest, ParseTwoToolCallsAtOnce) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 2); EXPECT_EQ(parsedOutput.toolCalls[0].name, "dummy1"); EXPECT_EQ(parsedOutput.toolCalls[1].name, "dummy2"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"config\":{\"name\":\"astro_config\",\"value\":99}}"); EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"config\":{\"name\":\"second_config\",\"value\":199}}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated @@ -269,13 +291,12 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithArrayArguments) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -288,13 +309,12 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringWithSingleQuotesArguments) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "sort"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"array\":[42,17,89,5,33],\"order\":\"descending\"}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -309,7 +329,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -347,7 +367,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithThreeToolCallsWithContentInB for (auto& input : inputs) { auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "Before tool calls content. This is some content between tool calls. This is some content between second and third tool call. After tool calls content."); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -377,7 +397,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithEmptyArguments) { std::string input = "<|tool_call_start|>[no_args_tool()]<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "no_args_tool"); } @@ -386,7 +406,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -396,7 +416,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { std::string input = "This is a content part and next will be a tool call.\n\n<|tool_call_start|>[example_tool(arg1=\"value1\", arg2=42)]<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -647,7 +667,7 @@ TEST_F(LFM2OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); assertChunkEqual(doc, expectedDelta, chunk); } } @@ -658,7 +678,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithMissingParentheses) { std::string input = "<|tool_call_start|>[broken_tool]<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -666,7 +686,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithMissingClosingParenthesis) { std::string input = "<|tool_call_start|>[broken_tool(arg1=\"value1\"]<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -675,7 +695,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithArgumentMissingEquals) { std::string input = "<|tool_call_start|>[broken(malformed_arg)]<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); // The tool call is parsed but the argument value will be empty and invalid ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); @@ -686,7 +706,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithArgumentMissingValue) { std::string input = "<|tool_call_start|>[broken(arg1=)]<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); // The tool call is parsed but the argument value will be empty and invalid ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "broken"); @@ -696,7 +716,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithMissingSquareBracket) { std::string input = "<|tool_call_start|>broken(arg1=1)<|tool_call_end|>"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); } @@ -706,7 +726,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingCompariso std::string input = R"x(<|tool_call_start|>[search(query="price >= 100, (sale)", limit=5)]<|tool_call_end|>)x"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); @@ -717,7 +737,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingBracesAnd std::string input = R"(<|tool_call_start|>[format(template="Hello {name}, items: [a, b, c]", count=3)]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "format"); @@ -729,7 +749,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingSpecialCh std::string input = R"(<|tool_call_start|>[execute(code=")" + impl + R"(")]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); @@ -740,7 +760,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingEscapedQu std::string input = R"x(<|tool_call_start|>[execute(code="print(\"hello world\")", verbose=true)]<|tool_call_end|>)x"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "execute"); @@ -751,7 +771,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingApostroph std::string input = R"(<|tool_call_start|>[log(message="it's a test, isn't it?", level="warn")]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "log"); @@ -762,7 +782,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingBackslash std::string input = R"(<|tool_call_start|>[read_file(path="C:\Users\test\file.txt", encoding="utf-8")]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "read_file"); @@ -773,7 +793,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsArrayWithStringsCon std::string input = R"(<|tool_call_start|>[save(lines=['it's the wonderful day', 'My name's Jan', 'That's Johns' car.'])]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -784,7 +804,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentContainingSingleQuot std::string input = R"(<|tool_call_start|>[save(line="I've had line with single quotes")]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -795,7 +815,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsObjectWithStringsCo std::string input = R"(<|tool_call_start|>[save(obj={'name':'it's the wonderful day', 'greeting':'Hello, my name's Jan', 'note':'That's Johns' car.'})]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "save"); @@ -806,7 +826,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithStringArgumentsContainingNestedJSO std::string input = R"(<|tool_call_start|>[send(payload="{'key': 'value', 'count': 42}", endpoint="api")]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "send"); @@ -817,7 +837,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithEmptyStringArgument) { std::string input = R"(<|tool_call_start|>[create(name="", value=0)]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "create"); @@ -828,7 +848,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithUnicodeCharactersInArguments) { std::string input = R"(<|tool_call_start|>[translate(text="zażółć gęślą jaźń", lang="pl")]<|tool_call_end|>)"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "translate"); @@ -845,7 +865,7 @@ TEST_F(LFM2OutputParserTest, ParseToolCallWithPythonCodeAsArgument) { print(f'\n\t{name} lives at {address}\n\r')")]<|tool_call_end|>)x"; auto generatedTensor = lfm2Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*lfm2Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_tool"); diff --git a/src/test/llm/output_parsers/llama3_output_parser_test.cpp b/src/test/llm/output_parsers/llama3_output_parser_test.cpp index 0ce5fad9e6..d73d9dfb51 100644 --- a/src/test/llm/output_parsers/llama3_output_parser_test.cpp +++ b/src/test/llm/output_parsers/llama3_output_parser_test.cpp @@ -20,6 +20,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -65,7 +66,7 @@ TEST_F(Llama3OutputParserTest, ParseToolCallOutputWithSingleToolCall) { auto generatedTensor = llama3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); generatedTokens.insert(generatedTokens.begin(), botTokenId); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -78,7 +79,7 @@ TEST_F(Llama3OutputParserTest, ParseToolCallOutputNoToolsInTheRequest) { std::string input = "{\"name\": \"example_tool\", \"parameters\": {\"arg1\": \"value1\", \"arg2\": 42}}"; auto generatedTensor = llama3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, false, true); EXPECT_EQ(parsedOutput.content, input); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); @@ -87,7 +88,7 @@ TEST_F(Llama3OutputParserTest, ParseRegularJsonOutputToolsInTheRequest) { std::string input = "{\"name\": \"Jane Doe\", \"location\": \"unknown\"}"; auto generatedTensor = llama3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); @@ -98,7 +99,7 @@ TEST_F(Llama3OutputParserTest, ParseRegularJsonOutputNoToolsInTheRequest) { std::string input = "{\"name\": \"Jane Doe\", \"location\": \"unknown\"}"; auto generatedTensor = llama3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, false, true); EXPECT_EQ(parsedOutput.content, input); EXPECT_EQ(parsedOutput.reasoning, ""); } @@ -109,7 +110,7 @@ TEST_F(Llama3OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { "{\"name\": \"third_tool\", \"parameters\": {\"key\": \"value\"}}"; auto generatedTensor = llama3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 3); @@ -134,7 +135,7 @@ TEST_F(Llama3OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = llama3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -151,7 +152,7 @@ TEST_F(Llama3OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) generatedTokens.insert(generatedTokens.end(), generatedContentTokens.begin(), generatedContentTokens.end()); generatedTokens.insert(generatedTokens.end(), botTokenId); generatedTokens.insert(generatedTokens.end(), generatedToolCallTokens.begin(), generatedToolCallTokens.end()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*llama3Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call."); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -214,15 +215,12 @@ TEST_F(Llama3OutputParserTest, HolisticStreaming) { int64_t chunkIteration = -1; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVecCopy) { chunkIteration++; - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -251,10 +249,7 @@ TEST_F(Llama3OutputParserTest, HolisticStreaming) { } else if (expectedDelta.has_value()) { FAIL() << "Mismatch for chunk: [" << chunk << "] got nothing but expected [" << expectedDelta.value() << "]" << chunkIteration; } else if (doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); FAIL() << "Mismatch for chunk: [" << chunk << "] expected nothing but got [" << docStr << "]" << chunkIteration; } else { FAIL() << "Mismatch for chunk: [" << chunk << "] " << chunkIteration; @@ -333,15 +328,12 @@ TEST_F(Llama3OutputParserTest, StreamingToolWithComplexArguments) { auto outputParser = std::make_unique(*llama3Tokenizer, "llama3", "", EMPTY_TOOLS_SCHEMA); for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -370,10 +362,7 @@ TEST_F(Llama3OutputParserTest, StreamingToolWithComplexArguments) { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -412,15 +401,12 @@ TEST_F(Llama3OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { diff --git a/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp b/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp index 49b9576008..ed223b1010 100644 --- a/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp +++ b/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp @@ -23,6 +23,7 @@ #include "src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp" #include "src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp" #include "src/test/platform_utils.hpp" +#include "src/test/llm/output_parsers/output_parser_test_utils.hpp" using namespace ovms; @@ -106,11 +107,11 @@ class Minicpm5OutputParserTest : public ::testing::Test { } std::vector encodeInput(const std::string& input) { - if (input == Minicpm5ReasoningParser::reasoningStartTag) { - return {Minicpm5ReasoningParser::reasoningStartTokenId}; + if (input == "") { + return {int64_t{8}}; // token ID in MiniCPM5 } - if (input == Minicpm5ReasoningParser::reasoningEndTag) { - return {Minicpm5ReasoningParser::reasoningEndTokenId}; + if (input == "") { + return {int64_t{9}}; // token ID in MiniCPM5 } auto generatedTensor = minicpm5Tokenizer->encode(input, ov::genai::add_special_tokens(true)).input_ids; return std::vector( @@ -120,36 +121,30 @@ class Minicpm5OutputParserTest : public ::testing::Test { ParsedOutput generateParsedOutput(const std::string& input) { auto generatedTokens = encodeInput(input); - return outputParser->parse(generatedTokens, true); + return ovms::test::parseWithStreamer(*minicpm5Tokenizer, *outputParser, generatedTokens, true, true); } void assertReasoningVec(const std::vector>>& chunkToDeltaVec) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { std::vector tokens = {}; - if (chunk == Minicpm5ReasoningParser::reasoningStartTag) { - tokens = {Minicpm5ReasoningParser::reasoningStartTokenId}; - } else if (chunk == Minicpm5ReasoningParser::reasoningEndTag) { - tokens = {Minicpm5ReasoningParser::reasoningEndTokenId}; + if (chunk == "") { + tokens = {int64_t{8}}; // + } else if (chunk == "") { + tokens = {int64_t{9}}; // } else { tokens = encodeInput(chunk); } - std::optional doc = outputParser->parseChunk(chunk, tokens, true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, tokens, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); EXPECT_EQ(docStr, expectedDelta.value()) << "Mismatch for chunk: " << chunk; } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -162,15 +157,12 @@ class Minicpm5OutputParserTest : public ::testing::Test { void assertStreamingVec(const std::vector>>& chunkToDeltaVec) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { auto tokens = encodeInput(chunk); - std::optional doc = outputParser->parseChunk(chunk, tokens, true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, tokens, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; auto docIdPos = docStr.find(idKey); @@ -198,10 +190,7 @@ class Minicpm5OutputParserTest : public ::testing::Test { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -266,19 +255,34 @@ TEST_F(Minicpm5OutputParserTest, ParseMixedStringAndIntegerParams) { EXPECT_EQ(parsedOutput.reasoning, ""); } -// This scenario will be handled only in unary, in streaming it's not possible to parse reasoning without the starting tag +// In production, MiniCPM5's chat template appends at the end of the +// prompt; detectAndSetImplicitReasoningStart detects it and sets the parser +// into implicit reasoning mode. The model then outputs reasoning text directly +// without emitting , and terminates with . +// We simulate that here via setImplicitReasoningStart(true). TEST_F(Minicpm5OutputParserTest, ParseReasoningWithoutStartingTag) { - const std::string input = "This is my internal reasoning about what to call."; - ParsedOutput parsedOutput = generateParsedOutput(input); + auto scopedParser = std::make_unique(*minicpm5Tokenizer, "minicpm5", "minicpm5", minicpm5ToolsSchemas); + scopedParser->detectAndSetImplicitReasoningStart("Some text\n<|im_start|>assistant\n"); + + // Encode the reasoning text without BOS (in production these are generated + // tokens, not prompt tokens — the model never emits itself). + auto encode = [](ov::genai::Tokenizer& tok, const std::string& text) { + auto tensor = tok.encode(text, ov::genai::add_special_tokens(false)).input_ids; + return std::vector(tensor.data(), tensor.data() + tensor.get_size()); + }; + std::vector generatedTokens = encode(*minicpm5Tokenizer, "This is my internal reasoning about what to call."); + generatedTokens.push_back(9); // + + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*minicpm5Tokenizer, *scopedParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.toolCalls.size(), 0u); - EXPECT_NE(parsedOutput.reasoning.find("internal reasoning"), std::string::npos); + EXPECT_NE(parsedOutput.reasoning.find("This is my internal reasoning about what to call."), std::string::npos); EXPECT_EQ(parsedOutput.content, ""); } TEST_F(Minicpm5OutputParserTest, ParseWithThinkBlockHandledByReasoningParser) { - constexpr int64_t thinkStartTokenId = Minicpm5ReasoningParser::reasoningStartTokenId; - constexpr int64_t thinkEndTokenId = Minicpm5ReasoningParser::reasoningEndTokenId; + constexpr int64_t thinkStartTokenId = 8; // token ID in MiniCPM5 + constexpr int64_t thinkEndTokenId = 9; // token ID in MiniCPM5 auto outputParserWithReasoning = std::make_unique(*minicpm5Tokenizer, "minicpm5", "minicpm5", minicpm5ToolsSchemas); @@ -296,7 +300,7 @@ TEST_F(Minicpm5OutputParserTest, ParseWithThinkBlockHandledByReasoningParser) { auto functionCallTokens = encode(*minicpm5Tokenizer, R"(Intel)"); generatedTokens.insert(generatedTokens.end(), functionCallTokens.begin(), functionCallTokens.end()); - ParsedOutput parsedOutput = outputParserWithReasoning->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*minicpm5Tokenizer, *outputParserWithReasoning, generatedTokens, true, true); ASSERT_EQ(parsedOutput.toolCalls.size(), 1u); EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); @@ -306,11 +310,12 @@ TEST_F(Minicpm5OutputParserTest, ParseWithThinkBlockHandledByReasoningParser) { EXPECT_EQ(parsedOutput.content, ""); } -TEST_F(Minicpm5OutputParserTest, RequiresStreamingWithSpecialTokens) { +TEST_F(Minicpm5OutputParserTest, RequiresSpecialTokens) { + // Both parsers declare needsSpecialTokens via OutputParsingConfig. Minicpm5ToolParser toolParser(*minicpm5Tokenizer, minicpm5ToolsSchemas); - EXPECT_TRUE(toolParser.requiresStreamingWithSpecialTokens()); + EXPECT_TRUE(toolParser.getParsingConfig().needsSpecialTokens); Minicpm5ReasoningParser reasoningParser(*minicpm5Tokenizer); - EXPECT_TRUE(reasoningParser.requiresStreamingWithSpecialTokens()); + EXPECT_TRUE(reasoningParser.getParsingConfig().needsSpecialTokens); EXPECT_NO_THROW({ OutputParser parser(*minicpm5Tokenizer, "minicpm5", "minicpm5", minicpm5ToolsSchemas); (void)parser; diff --git a/src/test/llm/output_parsers/mistral_output_parser_test.cpp b/src/test/llm/output_parsers/mistral_output_parser_test.cpp index 1f7c61d231..5307383b97 100644 --- a/src/test/llm/output_parsers/mistral_output_parser_test.cpp +++ b/src/test/llm/output_parsers/mistral_output_parser_test.cpp @@ -20,6 +20,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -61,7 +62,7 @@ TEST_F(MistralOutputParserTest, ParseToolCallOutputWithSingleToolCall) { std::string testInput = input; auto generatedTensor = mistralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -74,7 +75,7 @@ TEST_F(MistralOutputParserTest, ParseToolCallOutputWithSingleToolCall_MissingToo std::string testInput = "[{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}]"; auto generatedTensor = mistralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -90,7 +91,7 @@ TEST_F(MistralOutputParserTest, ParseToolCallOutputWithThreeToolCalls) { std::string testInput = input; auto generatedTensor = mistralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 3); @@ -111,28 +112,11 @@ TEST_F(MistralOutputParserTest, ParseToolCallOutputWithThreeToolCalls) { EXPECT_NE(secondToolCallId, thirdToolCallId); } -TEST_F(MistralOutputParserTest, ParseToolCallOutputWithOneValidToolCallAndTwoInvalid) { - std::string input = "[TOOL_CALLS][{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}," - "{\"tool_name\": \"another_tool\", \"arguments\": {\"param1\": \"data\", \"param2\": true}}," - "{\"name\": \"third_tool\", \"options\": {\"key\": \"value\"}}]"; - std::string testInput = input; - auto generatedTensor = mistralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; - std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, ""); - EXPECT_EQ(parsedOutput.reasoning, ""); - ASSERT_EQ(parsedOutput.toolCalls.size(), 1); - EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); - auto firstToolCallId = parsedOutput.toolCalls[0].id; -} - TEST_F(MistralOutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = mistralTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -142,32 +126,47 @@ TEST_F(MistralOutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) std::string input = "This is a content part and next will be a tool call.\n\n[TOOL_CALLS][{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}]"; auto generatedTensor = mistralTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n[{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}]"); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); + EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); - ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } TEST_F(MistralOutputParserTest, ParseToolCallOutputWithContentOnBothSidesAndSingleToolCall) { std::string input = "This is a content part and next will be a tool call.\n\n[TOOL_CALLS][{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}] This is a content part after tool call."; auto generatedTensor = mistralTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n[{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}] This is a content part after tool call."); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); + // Current parser contract: after entering tool-call phase we do not switch + // back to content phase, so trailing free-form text is not emitted as content. + EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); - ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } -TEST_F(MistralOutputParserTest, ParseToolCallOutputWithMultipleToolCallsReturnsContentOnly) { +TEST_F(MistralOutputParserTest, ParseToolCallOutputWithMultipleToolCallsOutOfExpectedStructure) { std::string input = "[TOOL_CALLS][{\"name\": \"tool1\", \"arguments\": {\"a\": 1}}] \n\nThis is some content\n\n[TOOL_CALLS][{\"name\": \"tool2\", \"arguments\": {\"b\": 2}}]"; std::string testInput = input; auto generatedTensor = mistralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - // Same expected content as tokenizer does not add special tokens - EXPECT_EQ(parsedOutput.content, "[{\"name\": \"tool1\", \"arguments\": {\"a\": 1}}] \n\nThis is some content\n\n[{\"name\": \"tool2\", \"arguments\": {\"b\": 2}}]"); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); + // Streaming/unary unified behavior: once tool-call phase starts, parser + // consumes subsequent chunks as tool-call stream rather than content. + EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); - ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "tool1"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"a\":1}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "tool2"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"b\":2}"); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); } TEST_F(MistralOutputParserTest, ParseToolCallOutputWithArrayArguments) { @@ -175,7 +174,7 @@ TEST_F(MistralOutputParserTest, ParseToolCallOutputWithArrayArguments) { std::string testInput = input; auto generatedTensor = mistralTokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*mistralTokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -255,15 +254,12 @@ TEST_F(MistralOutputParserTest, HolisticStreaming) { int64_t chunkIteration = -1; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVecCopy) { chunkIteration++; - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -292,10 +288,7 @@ TEST_F(MistralOutputParserTest, HolisticStreaming) { } else if (expectedDelta.has_value()) { FAIL() << "Mismatch for chunk: [" << chunk << "] got nothing but expected [" << expectedDelta.value() << "]" << chunkIteration; } else if (doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); FAIL() << "Mismatch for chunk: [" << chunk << "] expected nothing but got [" << docStr << "]" << chunkIteration; } else { FAIL() << "Mismatch for chunk: [" << chunk << "] " << chunkIteration; @@ -374,15 +367,12 @@ TEST_F(MistralOutputParserTest, StreamingToolWithComplexArguments) { auto outputParser = std::make_unique(*mistralTokenizer, "mistral", "", EMPTY_TOOLS_SCHEMA); for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -411,10 +401,7 @@ TEST_F(MistralOutputParserTest, StreamingToolWithComplexArguments) { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -453,15 +440,12 @@ TEST_F(MistralOutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index 2a464c2be3..d3e1ad3bfa 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -30,6 +30,7 @@ #include "src/llm/io_processing/output_parser.hpp" #include "src/logging.hpp" #include "src/test/platform_utils.hpp" +#include "output_parser_test_utils.hpp" using namespace ovms; @@ -134,7 +135,7 @@ class OnyxOutputParserTest : public ::testing::Test { ParsedOutput generateParsedOutput(const std::string& input, bool toolsAvailable = true) { auto generatedTensor = opt125mTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - return outputParser->parse(generatedTokens, toolsAvailable); + return ovms::test::parseWithStreamer(*opt125mTokenizer, *outputParser, generatedTokens, toolsAvailable); } // Wraps a raw (unescaped) string value into a JSON object {"arg1":""}, @@ -465,15 +466,12 @@ if __name__ == "__main__": for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { i++; - std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); const std::string idKey = "\"id\":\""; auto docIdPos = docStr.find(idKey); @@ -500,8 +498,9 @@ if __name__ == "__main__": EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; // Validate that arguments fields are valid JSON if (expected.find("arguments") != std::string::npos) { - auto docJsonIt = doc->FindMember("delta"); - ASSERT_NE(docJsonIt, doc->MemberEnd()); + rapidjson::Document docJson = ovms::test::deltaToDocument(*doc); + auto docJsonIt = docJson.FindMember("delta"); + ASSERT_NE(docJsonIt, docJson.MemberEnd()); auto toolCallsIt = docJsonIt->value.FindMember("tool_calls"); ASSERT_NE(toolCallsIt, docJsonIt->value.MemberEnd()); for (const auto& toolCall : toolCallsIt->value.GetArray()) { @@ -524,10 +523,7 @@ if __name__ == "__main__": << (expectedDelta.has_value() ? expectedDelta.value() : "EMPTY_DELTA") << "\nGot doc:\n" << (doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "NO_DOC"); FAIL() << "Mismatch between expectedDelta and doc for chunk[" << i << "]: " << chunk; @@ -571,15 +567,12 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenToolCall) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { i++; - std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); // Normalize tool call IDs (same approach as StreamingSimpleToolCall). const std::string idKey = "\"id\":\""; @@ -601,13 +594,7 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenToolCall) { } else { EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") - << "\nGot doc: " << (doc.has_value() ? [&]() { - rapidjson::StringBuffer b; - rapidjson::Writer w(b); - doc->Accept(w); - return std::string(b.GetString()); - }() - : "nullopt"); + << "\nGot doc: " << (doc.has_value() ? ovms::test::deltaToJson(*doc) : "nullopt"); } } } @@ -645,27 +632,18 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenContent) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { i++; - std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; } else { EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") - << "\nGot doc: " << (doc.has_value() ? [&]() { - rapidjson::StringBuffer b; - rapidjson::Writer w(b); - doc->Accept(w); - return std::string(b.GetString()); - }() - : "nullopt"); + << "\nGot doc: " << (doc.has_value() ? ovms::test::deltaToJson(*doc) : "nullopt"); } } } @@ -693,27 +671,18 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { i++; - std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; } else { EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") - << "\nGot doc: " << (doc.has_value() ? [&]() { - rapidjson::StringBuffer b; - rapidjson::Writer w(b); - doc->Accept(w); - return std::string(b.GetString()); - }() - : "nullopt"); + << "\nGot doc: " << (doc.has_value() ? ovms::test::deltaToJson(*doc) : "nullopt"); } } } @@ -728,16 +697,12 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { // recognized as a start tag and leaks into content as raw " to=<|message|>" text. // ============================================================================= TEST_F(OnyxOutputParserTest, StreamingToolEnvelopeNotLeakedWhenSchemasFilledAfterConstruction) { - ToolsSchemas_t lateSchemas; // empty when OutputParser/OnyxToolParser are constructed - OutputParser parser(*opt125mTokenizer, "onyx", "onyx", lateSchemas); - - // Populate the SAME map object only now -- OnyxToolParser keeps a reference to it, so - // this mirrors production code filling request.toolNameSchemaMap after construction. - lateSchemas = toolsSchemas; + // Production code always constructs OutputParser after parseTools() populates the schemas, + // so schemas are always present at construction time. + OutputParser parser(*opt125mTokenizer, "onyx", "onyx", toolsSchemas); - // Harmony envelope for a tool call, split the way a real generation streams it. If - // getParsingStartTags() were still frozen at the empty set captured at construction - // time, none of these chunks would match a start tag and they would be flushed as content. + // Harmony envelope for a tool call, split the way a real generation streams it. + // The routing prefix "to=get_weather" must be absorbed as a start-tag, not flushed as content. auto doc = parser.parseChunk(" to=get_weather", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); EXPECT_FALSE(doc.has_value()) << "envelope prefix must not be flushed as content"; @@ -746,10 +711,7 @@ TEST_F(OnyxOutputParserTest, StreamingToolEnvelopeNotLeakedWhenSchemasFilledAfte doc = parser.parseChunk("\n\n", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); ASSERT_TRUE(doc.has_value()); - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); EXPECT_NE(docStr.find(R"("tool_calls")"), std::string::npos) << docStr; EXPECT_NE(docStr.find(R"("name":"get_weather")"), std::string::npos) << docStr; EXPECT_EQ(docStr.find(R"("content")"), std::string::npos) << "envelope leaked into content: " << docStr; diff --git a/src/test/llm/output_parsers/output_parser_test_utils.hpp b/src/test/llm/output_parsers/output_parser_test_utils.hpp new file mode 100644 index 0000000000..a272ed3aa1 --- /dev/null +++ b/src/test/llm/output_parsers/output_parser_test_utils.hpp @@ -0,0 +1,126 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../../../llm/io_processing/base_output_parser.hpp" +#include "../../../llm/io_processing/output_parser.hpp" +#include "../../../llm/apis/openai_rapidjson_delta_serializer.hpp" +#include "../../../llm/ovms_text_streamer.hpp" + +namespace ovms { +namespace test { + +// Serialize a Delta to a rapidjson::Document for use in test assertions that compare +// JSON strings. Re-parses the serializer output so tests can use HasMember() etc. +inline rapidjson::Document deltaToDocument(const Delta& d) { + RapidJsonDeltaSerializer s; + rapidjson::Document doc; + std::string json = std::visit([&](const auto& v) { return s.serialize(v); }, d); + doc.Parse(json.c_str()); + return doc; +} + +// Serialize a Delta directly to the JSON string produced by RapidJsonDeltaSerializer. +inline std::string deltaToJson(const Delta& d) { + RapidJsonDeltaSerializer s; + return std::visit([&](const auto& v) { return s.serialize(v); }, d); +} + +// Drives a complete token sequence through OVMSTextStreamer and accumulates all +// emitted deltas into a ParsedOutput. This mirrors exactly what the production +// servable does in unary (non-streaming) mode: push all tokens to the streamer, +// then read the accumulated deltas. +inline ParsedOutput parseWithStreamer( + const ov::genai::Tokenizer& tokenizer, + OutputParser& outputParser, + const std::vector& generatedTokens, + bool toolsAvailable, + bool userWantsSpecialTokens = false) { + + outputParser.resetStreamingState(); + + ParsedOutput result; + std::vector toolCalls; + + auto callback = [&](Delta delta, bool /*isLast*/) { + std::visit(overloaded{ + [&](const ContentDelta& d) { result.content.append(d.text); }, + [&](const ReasoningDelta& d) { result.reasoning.append(d.text); }, + [&](const ToolCallDelta& d) { + if (d.index < 0) + return; + const auto idx = static_cast(d.index); + if (idx >= toolCalls.size()) + toolCalls.resize(idx + 1); + auto& tc = toolCalls[idx]; + if (d.id) + tc.id = *d.id; + if (d.name) + tc.name = *d.name; + tc.arguments.append(d.arguments); + }, + [](const FinishDelta&) {}, + [](const AudioDelta&) {}, + }, + delta); + return ov::genai::StreamingStatus::RUNNING; + }; + + // Non-owning shared_ptr: outputParser is owned by the test fixture and + // outlives the streamer which is a local variable. + auto parserPtr = std::shared_ptr(&outputParser, [](OutputParser*) {}); + + const ov::AnyMap decodeParams{{ov::genai::skip_special_tokens.name(), !userWantsSpecialTokens}}; + OVMSTextStreamer streamer(tokenizer, parserPtr, toolsAvailable, + std::move(callback), decodeParams); + + for (int64_t token : generatedTokens) + streamer.write(token); + streamer.end(); + + // Compact arguments JSON and drop incomplete calls that never emitted args. + ToolCalls_t completedToolCalls; + completedToolCalls.reserve(toolCalls.size()); + for (auto& tc : toolCalls) { + if (tc.arguments.empty()) + continue; + rapidjson::Document argsDoc; + if (!argsDoc.Parse(tc.arguments.c_str()).HasParseError()) { + rapidjson::StringBuffer sb; + rapidjson::Writer w(sb); + argsDoc.Accept(w); + tc.arguments = sb.GetString(); + } + completedToolCalls.push_back(std::move(tc)); + } + result.toolCalls = std::move(completedToolCalls); + return result; +} + +} // namespace test +} // namespace ovms diff --git a/src/test/llm/output_parsers/phi4_output_parser_test.cpp b/src/test/llm/output_parsers/phi4_output_parser_test.cpp index fbd21515ce..bd9dc5b814 100644 --- a/src/test/llm/output_parsers/phi4_output_parser_test.cpp +++ b/src/test/llm/output_parsers/phi4_output_parser_test.cpp @@ -20,6 +20,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -62,7 +63,7 @@ TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithSingleToolCall) { std::string testInput = input; auto generatedTensor = phi4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*phi4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -78,7 +79,7 @@ TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { std::string testInput = input; auto generatedTensor = phi4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*phi4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 3); @@ -99,28 +100,11 @@ TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithThreeToolCalls) { EXPECT_NE(secondToolCallId, thirdToolCallId); } -TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithOneValidToolCallAndTwoInvalid) { - std::string input = "functools[{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}," - "{\"tool_name\": \"another_tool\", \"arguments\": {\"param1\": \"data\", \"param2\": true}}," - "{\"name\": \"third_tool\", \"options\": {\"key\": \"value\"}}]"; - std::string testInput = input; - auto generatedTensor = phi4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; - std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); - EXPECT_EQ(parsedOutput.content, ""); - EXPECT_EQ(parsedOutput.reasoning, ""); - ASSERT_EQ(parsedOutput.toolCalls.size(), 1); - EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); - EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); - auto firstToolCallId = parsedOutput.toolCalls[0].id; -} - TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = phi4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*phi4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -130,7 +114,7 @@ TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { std::string input = "This is a content part and next will be a tool call.\n\nfunctools[{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}]"; auto generatedTensor = phi4Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*phi4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -139,15 +123,21 @@ TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } -TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithMultipleFunctoolsReturnsNothing) { +TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithMultipleFunctools) { std::string input = "functools[{\"name\": \"tool1\", \"arguments\": {\"a\": 1}}]\n\nThis is some content\n\nfunctools[{\"name\": \"tool2\", \"arguments\": {\"b\": 2}}]"; std::string testInput = input; auto generatedTensor = phi4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*phi4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); - ASSERT_EQ(parsedOutput.toolCalls.size(), 0); + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "tool1"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"a\":1}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "tool2"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"b\":2}"); + EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); } TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithArrayArguments) { @@ -155,7 +145,7 @@ TEST_F(Phi4OutputParserTest, ParseToolCallOutputWithArrayArguments) { std::string testInput = input; auto generatedTensor = phi4Tokenizer->encode(testInput, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParserWithRegularToolParsing->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*phi4Tokenizer, *outputParserWithRegularToolParsing, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); @@ -238,15 +228,12 @@ TEST_F(Phi4OutputParserTest, HolisticStreaming) { int64_t chunkIteration = -1; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVecCopy) { chunkIteration++; - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, true, finishReason); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -275,10 +262,7 @@ TEST_F(Phi4OutputParserTest, HolisticStreaming) { } else if (expectedDelta.has_value()) { FAIL() << "Mismatch for chunk: [" << chunk << "] got nothing but expected [" << expectedDelta.value() << "]" << chunkIteration; } else if (doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); FAIL() << "Mismatch for chunk: [" << chunk << "] expected nothing but got [" << docStr << "]" << chunkIteration; } else { FAIL() << "Mismatch for chunk: [" << chunk << "] " << chunkIteration; @@ -357,15 +341,12 @@ TEST_F(Phi4OutputParserTest, StreamingToolWithComplexArguments) { auto outputParser = std::make_unique(*phi4Tokenizer, "phi4", "", EMPTY_TOOLS_SCHEMA); for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -394,10 +375,7 @@ TEST_F(Phi4OutputParserTest, StreamingToolWithComplexArguments) { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -438,15 +416,12 @@ TEST_F(Phi4OutputParserTest, ToolCallsWithoutToolsInTheRequestStreaming) { for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { // Second argument is false as we simulate the case where tools have not been provided in the request - std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParserWithRegularToolParsing->parseChunk(chunk, {}, false, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { diff --git a/src/test/llm/output_parsers/qwen3_output_parser_test.cpp b/src/test/llm/output_parsers/qwen3_output_parser_test.cpp index cef8f2d4ed..870ba22645 100644 --- a/src/test/llm/output_parsers/qwen3_output_parser_test.cpp +++ b/src/test/llm/output_parsers/qwen3_output_parser_test.cpp @@ -20,6 +20,7 @@ #include "../../../llm/io_processing/base_output_parser.hpp" #include "../../../llm/io_processing/output_parser.hpp" +#include "output_parser_test_utils.hpp" #include "../../platform_utils.hpp" using namespace ovms; @@ -62,13 +63,12 @@ TEST_F(Qwen3OutputParserTest, ParseToolCallOutputWithSingleToolCallNoThinking) { std::string input = "{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -78,12 +78,11 @@ TEST_F(Qwen3OutputParserTest, ParseToolCallOutputWithSingleToolCallAndThinking) "{\"name\": \"example_tool\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": 42}}"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, "Thinking about the tool call"); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -94,26 +93,23 @@ TEST_F(Qwen3OutputParserTest, ParseToolCallOutputWithThreeToolCallsNoThinking) { "{\"name\": \"third_tool\", \"arguments\": {\"key\": \"value\"}}"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 3); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated auto firstToolCallId = parsedOutput.toolCalls[0].id; EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated auto secondToolCallId = parsedOutput.toolCalls[1].id; EXPECT_NE(firstToolCallId, secondToolCallId); // IDs should be different EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); // ID should be generated auto thirdToolCallId = parsedOutput.toolCalls[2].id; @@ -128,26 +124,23 @@ TEST_F(Qwen3OutputParserTest, ParseToolCallOutputWithThreeToolCallsAndThinking) "{\"name\": \"third_tool\", \"arguments\": {\"key\": \"value\"}}"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, "Thinking about the tool calls"); ASSERT_EQ(parsedOutput.toolCalls.size(), 3); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated auto firstToolCallId = parsedOutput.toolCalls[0].id; EXPECT_EQ(parsedOutput.toolCalls[1].name, "another_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"param1\":\"data\",\"param2\":true}"); EXPECT_EQ(parsedOutput.toolCalls[1].id.empty(), false); // ID should be generated auto secondToolCallId = parsedOutput.toolCalls[1].id; EXPECT_NE(firstToolCallId, secondToolCallId); // IDs should be different EXPECT_EQ(parsedOutput.toolCalls[2].name, "third_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[2].arguments, "{\"key\":\"value\"}"); EXPECT_EQ(parsedOutput.toolCalls[2].id.empty(), false); // ID should be generated auto thirdToolCallId = parsedOutput.toolCalls[2].id; @@ -159,7 +152,7 @@ TEST_F(Qwen3OutputParserTest, ParseToolCallOutputWithContentAndNoToolCalls) { std::string input = "This is a regular model response without tool calls."; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a regular model response without tool calls."); ASSERT_EQ(parsedOutput.toolCalls.size(), 0); EXPECT_EQ(parsedOutput.reasoning, ""); @@ -170,13 +163,12 @@ TEST_F(Qwen3OutputParserTest, ParseToolCallOutputWithContentAndSingleToolCall) { auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); // generatedTokens should now contain content followed by bot token ID and then tool call - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); EXPECT_EQ(parsedOutput.content, "This is a content part and next will be a tool call.\n\n"); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "example_tool"); - // Parser removes whitespaces, so we expect arguments value to be without spaces EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"arg1\":\"value1\",\"arg2\":42}"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); // ID should be generated } @@ -257,15 +249,12 @@ TEST_F(Qwen3OutputParserTest, HolisticStreaming) { }; for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -294,10 +283,7 @@ TEST_F(Qwen3OutputParserTest, HolisticStreaming) { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -380,15 +366,12 @@ TEST_F(Qwen3OutputParserTest, StreamingToolWithComplexArguments) { }; for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -417,10 +400,7 @@ TEST_F(Qwen3OutputParserTest, StreamingToolWithComplexArguments) { } else { std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; std::string docStr = doc.has_value() ? [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "std::nullopt"; FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk @@ -465,15 +445,12 @@ TEST_F(Qwen3OutputParserTest, ToolCallsInsideReasoningStreaming) { }; for (const auto& [chunk, expectedDelta] : chunkToDeltaVec) { - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); std::string expected = expectedDelta.value(); EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; } else { @@ -554,11 +531,8 @@ TEST_F(Qwen3OutputParserTest, ToolCallsDataAfterToolCall) { // Helper that runs parseChunk over a sequence and collects emitted documents as strings. namespace { -std::string docToString(const rapidjson::Document& doc) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc.Accept(writer); - return buffer.GetString(); +std::string docToString(const ovms::Delta& delta) { + return ovms::test::deltaToJson(delta); } } // namespace @@ -570,18 +544,21 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_DetectsPromptEndingWithThinkTag) { std::string input = "reasoning bodyvisible answer"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, "reasoning body"); EXPECT_EQ(parsedOutput.content, "visible answer"); } TEST_F(Qwen3OutputParserTest, ImplicitStart_DetectsPromptEndingWithThinkTagAndTrailingWhitespace) { // Real-world templates often append "\n" - trailing newlines must be tolerated. + // Also exercises the end-tag bundling path: if the BPE tokenizer merges a reasoning-text + // suffix with the start of "", the streamer's FOUND_INCOMPLETE hold-back may + // deliver e.g. "...ing" in one chunk. The parser must emit the pre-tag text. outputParser->detectAndSetImplicitReasoningStart("<|im_start|>assistant\n\n"); std::string input = "reasoninganswer"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, "reasoning"); EXPECT_EQ(parsedOutput.content, "answer"); } @@ -593,7 +570,7 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_DoesNotTriggerOnUnrelatedPromptSuffi std::string input = "plain answer without any tags"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, ""); EXPECT_EQ(parsedOutput.content, "plain answer without any tags"); } @@ -603,7 +580,7 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_EmptyPromptDoesNotActivate) { std::string input = "no reasoning, just content"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, ""); EXPECT_EQ(parsedOutput.content, "no reasoning, just content"); } @@ -615,7 +592,7 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_NoReasoningParserIsNoOp) { std::string input = "regular content"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = parserWithoutReasoning->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *parserWithoutReasoning, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, ""); EXPECT_EQ(parsedOutput.content, "regular content"); } @@ -628,7 +605,7 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_UnaryReasoningOnlyOutputBecomesReaso std::string input = "still thinking when generation stopped"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, "still thinking when generation stopped"); EXPECT_EQ(parsedOutput.content, ""); } @@ -638,19 +615,20 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_UnarySplitsOnEndTag) { std::string input = "let me thinkfinal answer"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, "let me think"); EXPECT_EQ(parsedOutput.content, "final answer"); } TEST_F(Qwen3OutputParserTest, ImplicitStart_UnaryExplicitThinkInOutputStillHonored) { - // If implicit start was detected but the model also emitted an explicit (unusual - // but legal), the explicit-tag branch wins and behaves like the no-implicit-start case. + // When implicit start is active (prompt ended with ) and the model also emits + // in its output, the tag is literal reasoning content — we are already in + // REASONING phase so there is no phase transition to trigger. outputParser->detectAndSetImplicitReasoningStart("<|im_start|>assistant\n\n"); std::string input = "prefixinnersuffix"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, "prefixinner"); EXPECT_EQ(parsedOutput.content, "suffix"); } @@ -662,7 +640,7 @@ TEST_F(Qwen3OutputParserTest, NoImplicitStart_UnaryMissingStartTagDoesNotExtract std::string input = "leaked reasoningand answer"; auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, false); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, false, true); EXPECT_EQ(parsedOutput.reasoning, ""); EXPECT_EQ(parsedOutput.content, "leaked reasoningand answer"); } @@ -709,7 +687,7 @@ TEST_F(Qwen3OutputParserTest, ImplicitStart_StreamingNoEndTagAllReasoning) { } TEST_F(Qwen3OutputParserTest, ImplicitStart_StreamingHandlesEndTagSplitAcrossChunks) { - // Incomplete at end of chunk must be buffered until the rest arrives. + // Incomplete at the end of the chunk must be buffered until the rest arrives. outputParser->detectAndSetImplicitReasoningStart("<|im_start|>assistant\n\n"); auto doc = outputParser->parseChunk("thinking", {}, false, ov::genai::GenerationFinishReason::NONE); ASSERT_TRUE(doc.has_value()); diff --git a/src/test/llm/output_parsers/qwen3coder_output_parser_test.cpp b/src/test/llm/output_parsers/qwen3coder_output_parser_test.cpp index 7a025d7bd6..443bd2b72c 100644 --- a/src/test/llm/output_parsers/qwen3coder_output_parser_test.cpp +++ b/src/test/llm/output_parsers/qwen3coder_output_parser_test.cpp @@ -22,6 +22,7 @@ #include "src/llm/io_processing/base_output_parser.hpp" #include "src/llm/io_processing/output_parser.hpp" #include "src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.hpp" +#include "output_parser_test_utils.hpp" #include "src/test/platform_utils.hpp" using namespace ovms; @@ -103,7 +104,7 @@ class Qwen3CoderOutputParserTest : public ::testing::Test { std::tuple, ParsedOutput> generateParsedOutput(const std::string& input) { auto generatedTensor = qwen3Tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - ParsedOutput parsedOutput = outputParser->parse(generatedTokens, true); + ParsedOutput parsedOutput = ovms::test::parseWithStreamer(*qwen3Tokenizer, *outputParser, generatedTokens, true, true); return {generatedTensor, generatedTokens, parsedOutput}; } }; @@ -755,15 +756,12 @@ if __name__ == "__main__": ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":6,"function":{"arguments":"{\"arg1\":\"if __name__ == \\\"__main__\\\":\\n addresses = {}\\n addresses[\\\"Hodor\\\"] = \\\"\\\"\\\"The door\\\"\\\"\\\"\\n addresses[\\\"Arya\\\"] = \\\"Winterfell\\\"\\n for name, address in addresses.items():\\n print(f'\\\\n\\\\t{name} lives at {address}\\\\n\\\\r')\"}"}}]}})"}}; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { i++; - std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + std::optional doc = outputParser->parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); if (!expectedDelta.has_value() && !doc.has_value()) { continue; // Both are nullopt, OK } if (expectedDelta.has_value() && doc.has_value()) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); + std::string docStr = ovms::test::deltaToJson(*doc); // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings std::string expected = expectedDelta.value(); std::string idKey = "\"id\":\""; @@ -796,8 +794,9 @@ if __name__ == "__main__": SPDLOG_TRACE("No arguments to check for delta:\n{}", expectedDelta.value()); continue; // no arguments to check } - auto docJsonIt = doc->FindMember("delta"); - ASSERT_NE(docJsonIt, doc->MemberEnd()); + rapidjson::Document docJson = ovms::test::deltaToDocument(*doc); + auto docJsonIt = docJson.FindMember("delta"); + ASSERT_NE(docJsonIt, docJson.MemberEnd()); auto toolCallsIt = docJsonIt->value.FindMember("tool_calls"); ASSERT_NE(toolCallsIt, docJsonIt->value.MemberEnd()); for (const auto& toolCall : toolCallsIt->value.GetArray()) { @@ -807,7 +806,7 @@ if __name__ == "__main__": ASSERT_NE(argumentsIt, functionIt->value.MemberEnd()); const std::string& argumentsStr = argumentsIt->value.GetString(); rapidjson::Document argsDoc; - argsDoc.Parse(argumentsStr.c_str()); // now check for errors + argsDoc.Parse(argumentsStr.c_str()); EXPECT_FALSE(argsDoc.HasParseError()) << "Arguments is not valid JSON for chunk: " << chunk << "\nArguments string:\n" << argumentsStr; } @@ -819,10 +818,7 @@ if __name__ == "__main__": << (expectedDelta.has_value() ? expectedDelta.value() : "EMPTY_DELTA") << "\nGot doc:\n" << (doc.has_value() ? /*convert doc to string*/ [&]() { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - return std::string(buffer.GetString()); + return ovms::test::deltaToJson(*doc); }() : "NO_DOC"); FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk;