{ "schema_version": "0.1", "canitoolcall_version": "0.1.0.dev0", "engine": { "name": "ollama", "version": "7af39318", "commit": "7af393188defd52d370464de0d2064649cab9b41", "details": { "go_version": "go1.26.0", "llama_cpp_build": "b11081", "llama_cpp_commit": "161755f", "runner": "llama-server token rendering (preserved_tokens) + model/parsers via server/routes.go flow", "pinned_version": "7af393188defd52d370464de0d2064649cab9b41", "tokens_per_step": "one" } }, "run": { "started_at": "2026-09-26T05:36:43Z", "finished_at": "2026-09-26T05:36:50Z", "platform": "linux-x86_64", "python": "3.12.3", "fixtures_digest": "a1e7b55367e195e8aa6e2ef746b2cc141b44f37caad0887ff8f30b104616f836", "strategies": [ "one", "special", "token", "rand:1:8", "rand:2:8", "rand:3:8", "rand:4:8", "rand:5:8" ], "normalization": "soft-v1", "synthetic_strategies": [ "one", "special", "rand:1:8", "rand:2:8", "rand:3:8", "rand:4:8", "rand:5:8" ] }, "cases": [ { "fixture_id": "deepseek/vllm-v3-no-tool-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v3-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v3-parallel-no-newline", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v3-various-data-types", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v3-escaped-strings", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v31-text-before-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/vllm-v41-text-before-parallel-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v41-reasoning-text-before-parallel-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/sglang-v32-text-before-empty-invoke", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/sglang-v4-self-closing-invoke", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v3-malformed-missing-brace", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/vllm-v3-malformed-missing-call-tokens", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v3-nested-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v31-single-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-parallel-two-calls", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-text-before-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-empty-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-numeric-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-unicode-emoji", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-marker-in-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-truncated-mid-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-multi-turn-after-tool-result", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v31-nested-json", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "deepseek3", "parser_source": "create/metadata.go: deepseek architectures -> deepseek3, which parses the V3.1 tool format", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>", "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>", "<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "deepseek-ai--DeepSeek-V3.1.vocab.gguf", "vocab_gguf_sha256": "346cae28cfa6863405f8e2fecdc8c6f101249fdd9cf7bc8f235bb87bf40009e7", "vocab_gguf_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "deepseek/v32-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v32-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-multi-turn-after-tool-result", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v4-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-multi-turn-after-tool-result", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-long-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "deepseek/v41-whitespace-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "gemma4/single-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/single-call-thinking", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/no-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/no-call-thinking", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/text-after-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/text-after-call-thinking", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/empty-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/empty-arguments-thinking", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/bug-ollama-18390-key-with-spaces", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['create_workflow'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/nested-objects-and-arrays", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/edge-values-key-with-space", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['record'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/dictsort-case-insensitive-order", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/marker-in-arguments", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got 'and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\":\"Wrap calls as \\\\u003c|tool_call\\\\u003ecall:f{\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains '<|\"|>'; content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "streams": { "one": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "special": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "token": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "rand:1:8": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "rand:2:8": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "rand:3:8": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "rand:4:8": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null }, "rand:5:8": { "content": "and close thoughts with .<|\"|>}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\":\"Wrap calls as \\u003c|tool_call\\u003ecall:f{\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/unicode-emoji", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/numeric-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/string-with-newlines-and-backslashes", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/json-text-in-string", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/array-of-strings", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/long-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/array-of-objects-with-braces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/many-strings-then-string-array", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['submit'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/vllm-no-tool-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-single-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-multiple-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-text-before-call", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you. ', got 'Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you. ', got 'Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you. ', got 'Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you. ', got 'Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you. ', got 'Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you. ', got 'Let me check the weather for you.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "special": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "token": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/vllm-two-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-nested-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-number-and-boolean", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-hyphenated-function-name", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-dotted-function-name", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-no-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-reasoning-thought-prefix", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-reasoning-thought-prefix-multiline", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/sglang-text-before-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before ', got 'Some text before'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before ', got 'Some text before'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before ', got 'Some text before'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before ', got 'Some text before'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before ', got 'Some text before'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/sglang-text-around-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text before after'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text beforeafter'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text beforeafter'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text before after'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text beforeafter'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text beforeafter'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text before after'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text before after'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Some text before after', got 'Some text beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Some text before after', got 'Some text beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Some text before after', got 'Some text beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Some text before after', got 'Some text beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Some text before after', got 'Some text beforeafter'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": "Some text beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "special": { "content": "Some text beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "token": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": "Some text beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": "Some text beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": "Some text beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/sglang-nested-array-with-spaces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/sglang-nested-object", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/vllm-malformed-no-brace", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/multi-turn-final-answer", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/multi-turn-second-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/parallel-two-calls-thinking", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/parallel-three-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/parallel-mixed-with-empty", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/truncated-mid-string", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/truncated-in-name", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/truncated-before-close", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/truncated-in-reasoning", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gemma4/truncated-second-parallel-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "gemma4", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): gemma4", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "<|channel>", "", "<|tool_call>", "", "<|tool_response>", "<|\"|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "google--gemma-4-31B-it.vocab.gguf", "vocab_gguf_sha256": "c45a756342ca921303a5689e6ea3e4c5133e6ebfe79d158d91616c3597803e31", "vocab_gguf_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":\"\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-single-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-parallel-same-tool", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-parallel-mixed-nested", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-content-before-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-no-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-nested-object", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-unicode-emoji", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-empty-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-marker-in-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-schema-coercion", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-thinking-disabled", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-call-order-vs-tool-order", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm47-single-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-parallel-same-tool", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-content-before-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-no-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-nested-object", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-unicode-emoji", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-empty-arguments", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-marker-in-arguments", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\\\u003ctool_call\\\\u003e and\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "streams": { "one": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "special": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "token": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "rand:1:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "rand:2:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "rand:3:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "rand:4:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null }, "rand:5:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"docs/format.md\",\"content\":\"Wrap each call in \\u003ctool_call\\u003e and\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-typed-values", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-number-looking-string", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-schema-coercion", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-string-escapes", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-significant-whitespace", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"pad.txt\", \"content\": \" two spaces\\\\n\"}', got '{\"path\":\"pad.txt\",\"content\":\" two spaces\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Keep the spacing exactly.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"pad.txt\",\"content\":\" two spaces\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-long-arguments", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-json-looking-string", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-thinking-disabled", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-call-order-vs-tool-order", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-multi-turn-second-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm53-single-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-5.3.vocab.gguf", "vocab_gguf_sha256": "c2836d0d9c43ffb90b0cc3e8205ff5e78e7debb7f3ff2df1ada62c93223126a2", "vocab_gguf_revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm53-parallel-same-tool", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-5.3.vocab.gguf", "vocab_gguf_sha256": "c2836d0d9c43ffb90b0cc3e8205ff5e78e7debb7f3ff2df1ada62c93223126a2", "vocab_gguf_revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm53-no-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-5.3.vocab.gguf", "vocab_gguf_sha256": "c2836d0d9c43ffb90b0cc3e8205ff5e78e7debb7f3ff2df1ada62c93223126a2", "vocab_gguf_revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm53-schema-coercion", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-5.3.vocab.gguf", "vocab_gguf_sha256": "c2836d0d9c43ffb90b0cc3e8205ff5e78e7debb7f3ff2df1ada62c93223126a2", "vocab_gguf_revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm45-vllm-newline-format-content-before", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-vllm-newline-format-two-calls", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-vllm-coerces-schema-types", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-vllm-zero-arg-newline", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-vllm-with-think-tags", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm45-ollama-content-after-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm47-vllm-whitespace-preserved-in-value", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-vllm-content-before-no-space", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-vllm-whitespace-only-content", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-vllm-args-with-newlines", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-vllm-reasoning-without-open-tag", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-bug-three-calls-same-tool", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-truncated-inside-arg-value", "family": "glm", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-truncated-before-close-call", "family": "glm", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Berlin\",\"unit\":\"celsius\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-truncated-after-arg-key", "family": "glm", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "glm/glm47-truncated-in-reasoning", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "glm/glm47-missing-last-close-arg-value", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "glm-4.7", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): glm-4.7-flash; create/metadata.go glm4 -> glm-4.7", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "", "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "zai-org--GLM-4.7.vocab.gguf", "vocab_gguf_sha256": "6b9f8ab9bceda63102fc1fc9775e79287dd4c921eaefa03a9170ab4a14ae6570", "vocab_gguf_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-channel-first-recipient", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-channel-first-empty-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-split-deltas-constrain-nospace", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-call-on-analysis-channel", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-preamble-call-end-terminated", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-sequential-calls", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 1 ['get_weather']" } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\":\"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-call-then-final", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Done', got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Reasoning about query...", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"vllm\"}" } ], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-truncated-final", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-interrupted-final-only", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/vllm-malformed-headers", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/llamacpp-commentary-content-only", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-channel-analysis", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-channel-commentary", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-role-constrain", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/llamacpp-edit-code-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/llamacpp-builtin-python-recipient", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'print(\"hello\")\\', err=invalid character \\'p\\' looking for beginning of value'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'I will execute python to say hello'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "streams": { "one": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "special": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "token": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "rand:1:8": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "rand:2:8": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "rand:3:8": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "rand:4:8": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" }, "rand:5:8": { "content": null, "reasoning_content": "I will execute python to say hello", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='print(\"hello\")', err=invalid character 'p' looking for beginning of value" } } } }, { "fixture_id": "gpt-oss/llamacpp-stray-commentary-header", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "outcome exception \"OllamaError: error parsing tool call: raw='I'm\\nthinking', err=invalid character 'I' looking for beginning of value\"; accepted: ['no_tool_calls']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got \"Hello, world!\\nWhat's up?\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "streams": { "one": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "special": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "token": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "rand:1:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "rand:2:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "rand:3:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "rand:4:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" }, "rand:5:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='I'm\nthinking', err=invalid character 'I' looking for beginning of value" } } } }, { "fixture_id": "gpt-oss/harmony-test-channel-first-constrain-adjacent", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-test-role-recipient-constrain-adjacent", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-spec-receiving-tool-calls", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-spec-preamble-action-plan", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/bug-unknown-channel-comment", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/bug-garbled-channel-question-marks", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/bug-garbled-channel-commentary-question", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-single-call", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-single-call-no-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-compact-json", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-unicode-emoji", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-nested-json", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-empty-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-marker-in-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: error parsing tool call: raw=\\'{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\\\nHeader: <|start|>assista..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Write the Harmony cheat sheet to a file.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "streams": { "one": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "special": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "token": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "rand:1:8": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "rand:2:8": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "rand:3:8": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "rand:4:8": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" }, "rand:5:8": { "content": null, "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with <|return|>.\\nHeader: <|start|>assistant<|channel|>commentary to=functions.get_time <|constrain|>json<|message|>{}', err=unexpected end of JSON input" } } } }, { "fixture_id": "gpt-oss/harmony-marker-in-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-preamble-then-call", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-preamble-no-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-no-call-final", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-final-only", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-final-json-lookalike", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-string-escapes", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-numeric-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-long-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-multiline-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-content-type-json-text", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-no-content-type", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-truncated-in-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'The user asks for the weather in Paris. I should call get_weather.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "streams": { "one": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "special": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "token": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "rand:1:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "rand:2:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "rand:3:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "rand:4:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" }, "rand:5:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": "OllamaError: error parsing tool call: raw='{\"city\": \"Paris\", \"unit', err=unexpected end of JSON input" } } } }, { "fixture_id": "gpt-oss/harmony-truncated-in-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/harmony-truncated-in-header", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/hf-template-single-call", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/hf-template-unicode-nested", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/hf-template-empty-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/hf-template-no-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "gpt-oss/hf-template-final-answer", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "harmony", "parser_source": "server/routes.go shouldUseHarmony: gptoss family -> harmony", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "openai--gpt-oss-20b.vocab.gguf", "vocab_gguf_sha256": "41ab8478c7fafbb0012cdeda945237760bc83660a4c1c2772de03e0c867e5be9", "vocab_gguf_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "kimi/k2i-vllm-single-call-content-before", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-vllm-three-tool-calls", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-vllm-multiline-json", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-vllm-id-without-functions-prefix", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-vllm-noise-between-markers", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-vllm-empty-tool-section", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-vllm-content-after-tool-section", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-vllm-no-call-thinking-disabled", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-vllm-call-thinking-disabled", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-vllm-truncated-tools", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-vllm-reasoning-prefix-consumed", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-bug-truncated-reasoning-recorded", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-bug-response-only-completion", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-single-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-parallel-same-tool", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-parallel-three-tools", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-content-before-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-nested-json", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-unicode-emoji", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-empty-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-string-escapes", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-marker-in-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-marker-terminator-in-string", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-empty-think", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-thinking-disabled-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-multi-turn-second-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-single-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-content-before-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k2i-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-single-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-parallel-same-tool", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-content-before-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-nested-json", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-typed-values", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-empty-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-string-escapes", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-control-marker-text-in-value", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-xtml-attr-escaping", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-long-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-empty-think", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-thinking-disabled-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-thinking-disabled-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-multi-turn-second-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-truncated-inside-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-truncated-inside-argument", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k26-truncated-in-reasoning", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "kimi/k3-truncated-in-reasoning", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-vllm-parameters-key", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-vllm-arguments-key", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-vllm-deeply-nested", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-vllm-brackets-in-strings", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-vllm-escaped-quotes", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-vllm-json-without-name-is-content", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-sglang-python-tag-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-sglang-text-before-python-tag", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-vllm-pythonic-no-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-vllm-pythonic-more-types", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-vllm-pythonic-parameterless", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-vllm-pythonic-escaped-strings", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-vllm-pythonic-parallel-no-spaces", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-vllm-pythonic-python-start-end", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-bug-leading-json-object-in-content", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-bug-empty-json-object-in-content", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-bug-whole-call-single-delta", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-bug-leading-underscore-identifier", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-single-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-nested-json", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-unicode-emoji", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-empty-arguments", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-marker-in-arguments", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-string-escapes", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-typed-values", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-long-arguments", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-no-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-json-answer-not-a-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-multi-turn-second-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-meta-python-tag-type-function", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-meta-pythonic-parallel", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-meta-pythonic-int-and-string", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l4-truncated-missing-close-bracket", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "llama/l3-truncated-inside-parameters", "family": "llama", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v3-no-space", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v3-arguments-before-name", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v3-name-inside-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v3-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v3-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v3-malformed-not-json", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v11-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v11-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v11-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v13-think-no-end", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/vllm-v11-think-text", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/llamacpp-ministral3-reasoning-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got \"[THINK]I'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got \"[THINK]I'm\\nthinking[/THINK]\"; reasoning_content: nonstream \"I'm\\nthinking\", got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains '[THINK]'; content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Reasoning-2512.vocab.gguf", "vocab_gguf_sha256": "dde7ffb069c74ec4a86cf5a0ed23f6bad5e1e62d7a1936abf8fb6997d614e4dd", "vocab_gguf_revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/llamacpp-ministral3-parallel", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Reasoning-2512.vocab.gguf", "vocab_gguf_sha256": "dde7ffb069c74ec4a86cf5a0ed23f6bad5e1e62d7a1936abf8fb6997d614e4dd", "vocab_gguf_revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/llamacpp-ministral3-marker-in-reasoning", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '[THINK]Let me think about and more[/THINK]'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got '[THINK]Let me think about and more[/THINK]'; reasoning_content: nonstream 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: nonstream ['special_function'], got ['special_function', 'special_function']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains '[THINK]'; content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Reasoning-2512.vocab.gguf", "vocab_gguf_sha256": "dde7ffb069c74ec4a86cf5a0ed23f6bad5e1e62d7a1936abf8fb6997d614e4dd", "vocab_gguf_revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/llamacpp-v11-call-id", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/llamacpp-v11-call-id-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/llamacpp-devstral-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/bug-devstral-glob-braces-in-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-empty-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-marker-in-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-multiline-reasoning", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-truncated-in-reasoning", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-truncated-in-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13think-stop-at-open-marker", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v13-single-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-parallel-calls", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-content-before-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-no-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-nested-json", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-unicode-emoji", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-string-escapes", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-numeric-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-long-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-param-named-name", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v13-array-lookalike-argument", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "ministral", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): ministral-3, devstral-2", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "[TOOL_CALLS]", "[THINK]", "[/THINK]", "[ARGS]" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "mistralai--Ministral-3-14B-Instruct-2512.vocab.gguf", "vocab_gguf_sha256": "30b914c221986cd4072c94430a8b1ea19d80d171e3d6153d092e71105933243c", "vocab_gguf_revision": "29439f81c2be264d8d393273f99e7db9c0961120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "mistral/v11-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v11-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v11-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v11-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v3-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v3-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v3-empty-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "mistral/v3-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no built-in Ollama parser for this model at the pin; Ollama serves it through a Go template with the legacy tools parser or through llama-server's native chat (see the llamacpp engine); legacy template parser not covered" }, { "fixture_id": "qwen3-hermes/single-call-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"\\nThe user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"\\nThe user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"\\nThe user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"\\nThe user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"\\nThe user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"\\nThe user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Paris\",\"unit\":\"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/single-call-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/no-call-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "Hello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "special": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "token": { "content": "Hello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Hello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/no-call-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/text-before-call-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI need live weather data for London.', got 'I need live weather data for London.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI need live weather data for London.', got 'I need live weather data for London.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI need live weather data for London.', got 'I need live weather data for London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI need live weather data for London.', got 'I need live weather data for London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI need live weather data for London.', got 'I need live weather data for London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI need live weather data for London.', got 'I need live weather data for London.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": "\nI need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "special": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.", "reasoning_content": "\nI need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "\nI need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"London\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/text-before-call-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/empty-arguments-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/empty-arguments-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nget_time takes no arguments.', got 'get_time takes no arguments.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nget_time takes no arguments.', got 'get_time takes no arguments.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nget_time takes no arguments.', got 'get_time takes no arguments.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nget_time takes no arguments.', got 'get_time takes no arguments.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nget_time takes no arguments.', got 'get_time takes no arguments.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nget_time takes no arguments.', got 'get_time takes no arguments.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nget_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "\nget_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nget_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-mid-arguments", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-at-opener", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/nested-json-deep", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/unicode-emoji-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).', got 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).', got 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).', got 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).', got 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).', got 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).', got 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Translate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Translate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Translate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Translate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Translate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Translate the mixed-script greeting (日本語, العربية, emoji 🌸).", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\":\"こんにちは 🌸 مرحبا — naïve café 👩‍💻\",\"target\":\"de\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/marker-in-arguments", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: failed to parse JSON: unexpected end of JSON input'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: unexpected end of JSON input" } } } }, { "fixture_id": "qwen3-hermes/string-escapes-code", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/json-string-argument", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/numeric-arguments", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/long-arguments", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-plain-text-forwarded", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-bug-19056-boolean-argument", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-no-newlines-no-spaces", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-text-then-call-no-separator", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-two-calls-no-separator", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-content-and-call-single-chunk", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/sglang-text-before-call-with-space", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/sglang-qwen25-four-parallel-calls", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-reasoning-multiline", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-reasoning-without-start-token", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/instruct2507-single-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "create/metadata.go parserNameForIdentifier: qwen3 architectures -> qwen3 (no thinking)", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Instruct-2507.vocab.gguf", "vocab_gguf_sha256": "10332d6e429af871611cbcf65ef8d43f660a3020200cff1a0ec5ffbcfb51141a", "vocab_gguf_revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/instruct2507-parallel-calls", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "create/metadata.go parserNameForIdentifier: qwen3 architectures -> qwen3 (no thinking)", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Instruct-2507.vocab.gguf", "vocab_gguf_sha256": "10332d6e429af871611cbcf65ef8d43f660a3020200cff1a0ec5ffbcfb51141a", "vocab_gguf_revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/instruct2507-no-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "create/metadata.go parserNameForIdentifier: qwen3 architectures -> qwen3 (no thinking)", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Instruct-2507.vocab.gguf", "vocab_gguf_sha256": "10332d6e429af871611cbcf65ef8d43f660a3020200cff1a0ec5ffbcfb51141a", "vocab_gguf_revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/instruct2507-empty-arguments", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "create/metadata.go parserNameForIdentifier: qwen3 architectures -> qwen3 (no thinking)", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Instruct-2507.vocab.gguf", "vocab_gguf_sha256": "10332d6e429af871611cbcf65ef8d43f660a3020200cff1a0ec5ffbcfb51141a", "vocab_gguf_revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/instruct2507-text-before-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "create/metadata.go parserNameForIdentifier: qwen3 architectures -> qwen3 (no thinking)", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Instruct-2507.vocab.gguf", "vocab_gguf_sha256": "10332d6e429af871611cbcf65ef8d43f660a3020200cff1a0ec5ffbcfb51141a", "vocab_gguf_revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/vllm-invalid-json-missing-brace", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/sglang-malformed-json-in-tags", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "outcome exception \"OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: failed to parse JSON: invalid character 'o' in literal null (expecting 'u')" } } } }, { "fixture_id": "qwen3-hermes/multi-turn-second-call-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nIt is 20C in Zürich. Now search for cafés.', got 'It is 20C in Zürich. Now search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nIt is 20C in Zürich. Now search for cafés.', got 'It is 20C in Zürich. Now search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nIt is 20C in Zürich. Now search for cafés.', got 'It is 20C in Zürich. Now search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nIt is 20C in Zürich. Now search for cafés.', got 'It is 20C in Zürich. Now search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nIt is 20C in Zürich. Now search for cafés.', got 'It is 20C in Zürich. Now search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nIt is 20C in Zürich. Now search for cafés.', got 'It is 20C in Zürich. Now search for cafés.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "It is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "It is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "It is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "It is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "It is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "It is 20C in Zürich. Now search for cafés.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\":\"café Zürich\",\"filters\":{\"open_now\":true}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/multi-turn-final-answer-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe tool returned 20C.', got 'The tool returned 20C.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe tool returned 20C.', got 'The tool returned 20C.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe tool returned 20C.', got 'The tool returned 20C.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe tool returned 20C.', got 'The tool returned 20C.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe tool returned 20C.', got 'The tool returned 20C.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe tool returned 20C.', got 'The tool returned 20C.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "The tool returned 20C.", "tool_calls": [], "exception": null }, "special": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "The tool returned 20C.", "tool_calls": [], "exception": null }, "token": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "The tool returned 20C.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "The tool returned 20C.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "The tool returned 20C.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "It is 20 °C in Zürich right now.", "reasoning_content": "The tool returned 20C.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/parallel-two-calls-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tools.', got 'I should call the tools.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tools.', got 'I should call the tools.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tools.', got 'I should call the tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tools.', got 'I should call the tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tools.', got 'I should call the tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tools.', got 'I should call the tools.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nI should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "\nI should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nI should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\":\"Zürich\",\"unit\":\"c\"}" }, { "name": "search", "arguments_raw": "{\"query\":\"café \\\"best\\\"\",\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/parallel-three-calls-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/parallel-mixed-tools-text-before-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/thinking2507-single-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/thinking2507-parallel-calls", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/thinking2507-no-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/thinking2507-text-before-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/thinking2507-multi-turn", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/truncated-mid-arguments-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/truncated-mid-arguments-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tool.', got 'I should call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tool.', got 'I should call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tool.', got 'I should call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tool.', got 'I should call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tool.', got 'I should call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nI should call the tool.', got 'I should call the tool.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/truncated-after-open-tag", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/truncated-second-parallel-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-hermes/truncated-in-reasoning", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather', got 'The user wants the current weather'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather', got 'The user wants the current weather'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather', got 'The user wants the current weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather', got 'The user wants the current weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather', got 'The user wants the current weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather', got 'The user wants the current weather'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "hybrid Qwen3: qwen3-thinking when thinking is on (the Qwen3 parser that handles ), qwen3 when the request disables thinking", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": true, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-0.6B.vocab.gguf", "vocab_gguf_sha256": "720f095bcaff60b6e960fa8341d2cbb178dcef1bd3ef99e49c8b4e6174022451", "vocab_gguf_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user wants the current weather", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-truncated-in-reasoning", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-thinking", "parser_source": "model/parsers/parsers.go: qwen3-thinking = Qwen3Parser{thinking support, thinking by default}", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-4B-Thinking-2507.vocab.gguf", "vocab_gguf_sha256": "0e36cf4a350c75a450c9e1d15b54792f5f5233634ce6c8089863ed7e2ddcfc6b", "vocab_gguf_revision": "768f209d9ea81521153ed38c47d515654e938aea", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-reasoning-then-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-parallel-no-think", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-python-multiline", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-value-trailing-newline", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-code-leading-indent", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-call-inside-think", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n\\n\\n\\npwd\\n\\n\\n'; tool_calls: expected ['run_in_terminal'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n\\n\\n\\npwd\\n\\n\\n'; tool_calls: expected ['run_in_terminal'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n\\n\\n\\npwd\\n\\n\\n'; tool_calls: expected ['run_in_terminal'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n\\n\\n\\npwd\\n\\n\\n'; tool_calls: nonstream ['run_in_terminal'], got []" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n\\n\\n\\npwd\\n\\n\\n'; tool_calls: nonstream ['run_in_terminal'], got []" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n\\n\\n\\npwd\\n\\n\\n'; tool_calls: nonstream ['run_in_terminal'], got []" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''; reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''; reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''; reasoning_content contains ''; reasoning_content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-empty-args", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-q35-reasoning-content-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-coder-bare-function", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-coder-bare-function-parallel", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "streams": { "one": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "special": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "token": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "rand:1:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "rand:2:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "rand:3:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "rand:4:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null }, "rand:5:8": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\":1,\"arg2\":2}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-coder-unicode-cjk", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-coder-html-value", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/llamacpp-coder-array-of-objects", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/vllm-coder-text-before-call-no-newline", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/vllm-coder-anyof-coercion", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/vllm-coder-missing-close-parameter", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 11: element closed by '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got 'Let me check the weather for you:'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the weather for you:'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the weather for you:'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the weather for you:'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the weather for you:'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the weather for you:'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "token": { "content": "Let me check the weather for you:", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "rand:1:8": { "content": "Let me check the weather for you:", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "rand:2:8": { "content": "Let me check the weather for you:", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "rand:3:8": { "content": "Let me check the weather for you:", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "rand:4:8": { "content": "Let me check the weather for you:", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " }, "rand:5:8": { "content": "Let me check the weather for you:", "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 11: element closed by " } } } }, { "fixture_id": "qwen3-xml/bug-missing-close-parameter-before-function", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 5: element closed by '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 5: element closed by " } } } }, { "fixture_id": "qwen3-xml/bug-coder-missing-tool-call-opener", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.', got 'I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n'; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-xml/bug-coder-text-after-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/bug-coder-number-outside-int64", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": 1e+20}', got '{\"x\":-9223372036854775808}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\":-9223372036854775808}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-single-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-parallel-calls", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-three-parallel-calls", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-unicode-emoji", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-nested-json", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-empty-arguments", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-marker-in-arguments", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Write the note.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\":\"notes.md\",\"content\":\"Close tags inline: \"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-tool-call-token-in-arguments", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: unexpected EOF'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Document the format.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Document the format.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Document the format.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Document the format.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Document the format.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream None, got 'Document the format.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "token": { "content": null, "reasoning_content": "Document the format.", "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "rand:1:8": { "content": null, "reasoning_content": "Document the format.", "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "rand:2:8": { "content": null, "reasoning_content": "Document the format.", "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "rand:3:8": { "content": null, "reasoning_content": "Document the format.", "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "rand:4:8": { "content": null, "reasoning_content": "Document the format.", "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" }, "rand:5:8": { "content": null, "reasoning_content": "Document the format.", "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: unexpected EOF" } } } }, { "fixture_id": "qwen3-xml/q38-text-before-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-no-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-multiline-string", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-whitespace-significant", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-numeric-and-literal-arguments", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-numeric-looking-strings", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-string-escapes", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-long-arguments", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-empty-reasoning", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-thinking-disabled", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-thinking-disabled-parallel", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-thinking-disabled-no-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": false, "think_effective": false, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-truncated-in-reasoning", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-truncated-in-parameter", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q38-truncated-at-tool-call-open", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.8-27B.vocab.gguf", "vocab_gguf_sha256": "7b5dcc22c704d56bf35ebc9cc40ce3a2f766890c968f5c8b6d34a3808680749f", "vocab_gguf_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q35-python-literals", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/q35-single-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3.5", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3.5, qwen3.6; create/metadata.go qwen3_5", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": true, "preserved_tokens": [ "", "", "", "" ], "has_thinking_support": true, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3.5-9B.vocab.gguf", "vocab_gguf_sha256": "ccfbc41329c3caba178c629bf95b67eab472cee6f1a271ba69ce059056edd896", "vocab_gguf_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-single-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-parallel-calls", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-text-before-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-no-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-python-literals", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-empty-arguments", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-marker-in-arguments", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'OllamaError: XML syntax error on line 7: element closed by '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-30B-A3B-Instruct.vocab.gguf", "vocab_gguf_sha256": "835d0d2ad5a7f95016cdf13dffb330e8495936aac89fa4edf5e103c424ce9849", "vocab_gguf_revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "OllamaError: XML syntax error on line 7: element closed by " } } } }, { "fixture_id": "qwen3-xml/coder-next-json-literals", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-Next.vocab.gguf", "vocab_gguf_sha256": "5e0ca989989a8d2ad3462d205a1018a0085a9ce0540ed04debe504da48e69ba7", "vocab_gguf_revision": "a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } }, { "fixture_id": "qwen3-xml/coder-next-parallel-calls", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "parser": "qwen3-coder", "parser_source": "Ollama library model config (registry.ollama.ai, fetched 2026-09-25): qwen3-coder, qwen3-coder-next", "ollama_commit": "7af393188defd52d370464de0d2064649cab9b41", "think_requested": null, "think_effective": null, "preserved_tokens": [ "", "" ], "has_thinking_support": false, "detokenizer": "llama-server common_token_to_piece(special = id in preserved_tokens)", "llama_cpp_build": "b11081", "vocab_gguf": "Qwen--Qwen3-Coder-Next.vocab.gguf", "vocab_gguf_sha256": "5e0ca989989a8d2ad3462d205a1018a0085a9ce0540ed04debe504da48e69ba7", "vocab_gguf_revision": "a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb", "vocab_gguf_revision_matches_tokenizer": true, "vocab_gguf_converter_llama_cpp_commit": "a25c9865fe03c954c93fd755b5d79ae86ba99750", "chunking": "one llama-server event per token; Add(delta, false) per non-empty group, then Add('', true)", "eog_stop": "tokens after the first end-of-generation token are dropped (llama-server stops there)", "nonstream": "per-token events concatenated (routes.go writeChatResponse)", "tool_call_serialization": "openai.ToToolCalls", "legacy_template_parser": false } } ], "summary": { "totals": { "pass": 224, "soft_pass": 10, "fail": 27, "error": 0, "unsupported": 208 }, "by_family": { "deepseek": { "pass": 12, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 73 }, "gemma4": { "pass": 40, "soft_pass": 1, "fail": 7, "error": 0, "unsupported": 0 }, "glm": { "pass": 29, "soft_pass": 0, "fail": 3, "error": 0, "unsupported": 19 }, "gpt-oss": { "pass": 45, "soft_pass": 0, "fail": 6, "error": 0, "unsupported": 0 }, "kimi": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 49 }, "llama": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 34 }, "mistral": { "pass": 14, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 33 }, "qwen3-hermes": { "pass": 39, "soft_pass": 9, "fail": 2, "error": 0, "unsupported": 0 }, "qwen3-xml": { "pass": 45, "soft_pass": 0, "fail": 9, "error": 0, "unsupported": 0 } } } }