{
"schema_version": "0.1",
"canitoolcall_version": "0.1.0.dev0",
"engine": {
"name": "vllm",
"version": "0.30.0",
"commit": null,
"details": {
"transformers": "5.17.0",
"tokenizers": "0.23.2",
"torch": "2.14.0+cpu",
"openai-harmony": "0.0.8",
"mistral-common": "1.12.0",
"huggingface-hub": "1.33.0",
"wheel_sha256": "ef52ee58c410ead0b8afb190838fa4cbcb52075596f67862a03859d984966ac4",
"wheel": "vllm-0.30.0-cp38-abi3-manylinux_2_28_x86_64.whl",
"pinned_version": "0.30.0",
"tokens_per_step": "many"
}
},
"run": {
"started_at": "2026-09-26T05:19:00Z",
"finished_at": "2026-09-26T05:20:52Z",
"platform": "linux-x86_64",
"python": "3.12.14",
"fixtures_digest": "a1e7b55367e195e8aa6e2ef746b2cc141b44f37caad0887ff8f30b104616f836",
"strategies": [
"one",
"special",
"token",
"rand:1:8",
"rand:2:8",
"rand:3:8",
"rand:4:8",
"rand:5:8"
],
"normalization": "soft-v1",
"synthetic_strategies": []
},
"cases": [
{
"fixture_id": "deepseek/vllm-v3-no-tool-calls",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/vllm-v3-single-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v3-parallel-no-newline",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather', 'search_hotels'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather', 'search_hotels'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{\"city\": \"Tokyo\", \"unit\":'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather', 'search_hotels'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather', 'search_hotels'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{\"city\": \"Tokyo\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather', 'search_hotels'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather', 'search_hotels'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather', 'search_hotels'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather', 'search_hotels'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather', 'search_hotels'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather', 'search_hotels']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\", \"unit\":': Expecting value: line 1 column 26 (char 25))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "fail",
"detail": "expected 2 calls ['get_weather', 'search_hotels'], got 0 []"
},
{
"check": "parallel_order",
"strategy": "special",
"status": "fail",
"detail": "expected 2 calls ['get_weather', 'search_hotels'], got 0 []"
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\":"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}"
},
{
"name": "search_hotels",
"arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v3-various-data-types",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['test_function'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['test_function'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..."
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..."
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['test_function'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['test_function'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['test_function'] | rand:1:8 (fail): tool_calls: one [], got ['test_function'] | rand:2:8 (fail): tool_calls: one [], got ['test_function'] | rand:3:8 (fail): tool_calls: one [], got ['test_function'] | rand:4:8 (fail): tool_calls: one [], got ['test_function'] | rand:5:8 (fail): tool_calls: one [], got ['test_function']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] test_function: arguments are not valid JSON text ('{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne...: Expecting value: line 1 column 208 (char 207))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] test_function: arguments are not valid JSON text ('{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne...: Unterminated string starting at: line 1 column 193 (char 192))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] test_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] test_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\":"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "test_function",
"arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v3-escaped-strings",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['send_message'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['send_message'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['send_message'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['send_message'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['send_message'] | rand:1:8 (fail): tool_calls: one [], got ['send_message'] | rand:2:8 (fail): tool_calls: one [], got ['send_message'] | rand:3:8 (fail): tool_calls: one [], got ['send_message'] | rand:4:8 (fail): tool_calls: one [], got ['send_message'] | rand:5:8 (fail): tool_calls: one [], got ['send_message']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "send_message",
"arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v31-text-before-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected 'normal text', got None; tool_calls: expected ['foo'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected 'normal text', got None; tool_calls: expected ['foo'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"x\": 1}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"x\": 1}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected 'normal text', got None; tool_calls[0].arguments: expected '{\"x\": 1}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected 'normal text', got None"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected 'normal text', got None; tool_calls[0].arguments: expected '{\"x\": 1}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'normal text', got None; tool_calls: nonstream ['foo'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'normal text', got None; tool_calls: nonstream ['foo'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"x\":1}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"x\":1}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'normal text', got None; tool_calls[0].arguments: nonstream '{\"x\":1}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'normal text', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'normal text', got None; tool_calls[0].arguments: nonstream '{\"x\":1}', got ''"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): content: one None, got 'normal text'; tool_calls: one [], got ['foo'] | rand:1:8 (fail): content: one None, got 'normal text'; tool_calls: one [], got ['foo'] | rand:2:8 (fail): content: one None, got 'normal text'; tool_calls: one [], got ['foo'] | rand:3:8 (fail): tool_calls: one [], got ['foo'] | rand:4:8 (fail): tool_calls: one [], got ['foo'] | rand:5:8 (fail): tool_calls: one [], got ['foo']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] foo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] foo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] foo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] foo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] foo: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] foo: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] foo: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] foo: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "normal text",
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": "{\"x\":1}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "normal text",
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": "{\"x\":1}"
}
],
"exception": null
},
"rand:1:8": {
"content": "normal text",
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": "normal text",
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": "{\"x\":1}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "foo",
"arguments_raw": ""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v41-text-before-parallel-calls",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"special": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"token": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Checking.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v41-reasoning-text-before-parallel-calls",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Checking.', got 'Checking.\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"special": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"token": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Checking.\n\n",
"reasoning_content": "Plan.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"杭州\", \"count\": 42}"
},
{
"name": "add",
"arguments_raw": "{\"x\": 1.5, \"y\": 2.25}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/sglang-v32-text-before-empty-invoke",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "erties\": {}}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me get the current date for you.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_date",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/sglang-v4-self-closing-invoke",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['submit'], got ['submit\"/>\\n']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].name contains '｜DSML｜'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] 'submit\"/>\\n' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "submit\"/>\n",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v3-malformed-missing-brace",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"', got '{\"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"', got '{\"city\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"', got '{\"'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\"': Expecting ',' delimiter: line 1 column 17 (char 16))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\"': Expecting ',' delimiter: line 1 column 17 (char 16))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"': Unterminated string starting at: line 1 column 2 (char 1))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\":': Expecting value: line 1 column 9 (char 8))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\"': Expecting ',' delimiter: line 1 column 17 (char 16))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"': Unterminated string starting at: line 1 column 2 (char 1))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\""
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\":"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/vllm-v3-malformed-missing-call-tokens",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got 'function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"}\\n```'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"}\n```",
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-single-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-parallel-two-calls",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather', 'search'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather', 'search'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"c'; tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather', 'search'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather', 'search'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"c'; tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): content (whitespace only): one None, got '\\n'; tool_calls: one [], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather', 'search']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3': Expecting ',' delimiter: line 1 column 68 (char 67))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"c': Unterminated string starting at: line 1 column 28 (char 27)); [1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"': Unterminated string starting at: line 1 column 60 (char 59))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated; [1] search: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "fail",
"detail": "expected 2 calls ['get_weather', 'search'], got 0 []"
},
{
"check": "parallel_order",
"strategy": "special",
"status": "fail",
"detail": "expected 2 calls ['get_weather', 'search'], got 0 []"
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v3-text-before-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): content: one None, got 'Let me check the current'; tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): content: one None, got 'Let me check the current conditions in'; tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): content: one None, got 'Let me check the current'; tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-empty-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_time'] | rand:1:8 (fail): tool_calls: one [], got ['get_time'] | rand:2:8 (fail): tool_calls: one [], got ['get_time'] | rand:3:8 (fail): tool_calls: one [], got ['get_time'] | rand:4:8 (fail): tool_calls: one [], got ['get_time'] | rand:5:8 (fail): tool_calls: one [], got ['get_time']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-numeric-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['calculate'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['calculate'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['calculate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['calculate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['calculate'] | rand:1:8 (fail): tool_calls: one [], got ['calculate'] | rand:2:8 (fail): tool_calls: one [], got ['calculate'] | rand:3:8 (fail): tool_calls: one [], got ['calculate'] | rand:4:8 (fail): tool_calls: one [], got ['calculate'] | rand:5:8 (fail): tool_calls: one [], got ['calculate']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-unicode-emoji",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['translate'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['translate'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['translate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['translate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['translate'] | rand:1:8 (fail): tool_calls: one [], got ['translate'] | rand:2:8 (fail): tool_calls: one [], got ['translate'] | rand:3:8 (fail): tool_calls: one [], got ['translate'] | rand:4:8 (fail): tool_calls: one [], got ['translate'] | rand:5:8 (fail): tool_calls: one [], got ['translate']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-marker-in-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['echo'] | rand:1:8 (fail): tool_calls: one [], got ['echo'] | rand:2:8 (fail): tool_calls: one [], got ['echo'] | rand:3:8 (fail): tool_calls: one [], got ['echo'] | rand:4:8 (fail): tool_calls: one [], got ['echo'] | rand:5:8 (fail): tool_calls: one [], got ['echo']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-truncated-mid-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür'"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v3-nested-json",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['create_event'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['create_event'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['create_event'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['create_event'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['create_event'] | rand:1:8 (fail): tool_calls: one [], got ['create_event'] | rand:2:8 (fail): tool_calls: one [], got ['create_event'] | rand:3:8 (fail): tool_calls: one [], got ['create_event'] | rand:4:8 (fail): tool_calls: one [], got ['create_event'] | rand:5:8 (fail): tool_calls: one [], got ['create_event']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] create_event: arguments are not valid JSON text ('{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam...: Unterminated string starting at: line 1 column 195 (char 194))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] create_event: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v3",
"reasoning_parser": null,
"parser_source": "docs/formats/deepseek.md (V3: no reasoning)",
"model_rule": "DeepSeek-V3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3-0324",
"revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"requested_mode": "auto",
"resolved_revision": "e9b33add76883f293d6bf61f6bd89b497e80e335",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3-0324",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-single-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-parallel-two-calls",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather', 'search'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather', 'search'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"c'; tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"'; tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\":'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"],'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather', 'search'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather', 'search'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"c'; tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"'; tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"],'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather', 'search']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"': Unterminated string starting at: line 1 column 60 (char 59))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"c': Unterminated string starting at: line 1 column 28 (char 27)); [1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": ': Expecting value: line 1 column 67 (char 66))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3': Expecting ',' delimiter: line 1 column 68 (char 67))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27)); [1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\":': Expecting value: line 1 column 66 (char 65))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] search: arguments are not valid JSON text ('{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"],': Expecting property name enclosed in double quotes: line 1 column 59 (char 58))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated; [1] search: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] search: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated; [1] search: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] search: arguments are not a JSON object; not validated"
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "fail",
"detail": "expected 2 calls ['get_weather', 'search'], got 0 []"
},
{
"check": "parallel_order",
"strategy": "special",
"status": "fail",
"detail": "expected 2 calls ['get_weather', 'search'], got 0 []"
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \""
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\":"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"],"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v31-text-before-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in'; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): content: one None, got 'Let me check the current'; tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): content: one None, got 'Let me check the current conditions in'; tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): content: one None, got 'Let me check the current'; tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-empty-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_time'] | rand:1:8 (fail): tool_calls: one [], got ['get_time'] | rand:2:8 (fail): tool_calls: one [], got ['get_time'] | rand:5:8 (fail): tool_calls: one [], got ['get_time']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-numeric-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['calculate'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['calculate'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['calculate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['calculate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['calculate'] | rand:1:8 (fail): tool_calls: one [], got ['calculate'] | rand:2:8 (fail): tool_calls: one [], got ['calculate'] | rand:3:8 (fail): tool_calls: one [], got ['calculate'] | rand:4:8 (fail): tool_calls: one [], got ['calculate'] | rand:5:8 (fail): tool_calls: one [], got ['calculate']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "calculate",
"arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-unicode-emoji",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['translate'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['translate'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['translate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['translate'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['translate'] | rand:1:8 (fail): tool_calls: one [], got ['translate'] | rand:2:8 (fail): tool_calls: one [], got ['translate'] | rand:3:8 (fail): tool_calls: one [], got ['translate'] | rand:4:8 (fail): tool_calls: one [], got ['translate'] | rand:5:8 (fail): tool_calls: one [], got ['translate']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-marker-in-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['f']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['f']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['echo', 'f']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['echo', 'f']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['echo', 'f']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['echo', 'f']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['echo', 'f']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['echo'], got ['echo', 'f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['echo', 'f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['echo', 'f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['echo', 'f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['echo', 'f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['echo', 'f']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: nonstream ['echo'], got ['echo', 'f']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one ['f'], got ['echo', 'f'] | rand:1:8 (fail): tool_calls: one ['f'], got ['echo', 'f'] | rand:2:8 (fail): tool_calls: one ['f'], got ['echo', 'f'] | rand:3:8 (fail): tool_calls: one ['f'], got ['echo', 'f'] | rand:4:8 (fail): tool_calls: one ['f'], got ['echo', 'f'] | rand:5:8 (fail): tool_calls: one ['f'], got ['echo', 'f']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[1].arguments contains '<｜tool▁calls▁end｜>'; tool_calls[1].arguments contains '<｜tool▁call▁end｜>'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[1].arguments contains '<｜tool▁calls▁end｜>'; tool_calls[1].arguments contains '<｜tool▁call▁end｜>'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[1].arguments contains '<｜tool▁calls▁end｜>'; tool_calls[1].arguments contains '<｜tool▁call▁end｜>'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[1].arguments contains '<｜tool▁calls▁end｜>'; tool_calls[1].arguments contains '<｜tool▁call▁end｜>'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[1].arguments contains '<｜tool▁calls▁end｜>'; tool_calls[1].arguments contains '<｜tool▁call▁end｜>'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] f: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] f: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like ': Unterminated string starting at: line 1 column 10 (char 9)); [1] f: arguments are not valid JSON text ('{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>': Extra data: line 1 column 4 (char 3))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like': Unterminated string starting at: line 1 column 10 (char 9)); [1] f: arguments are not valid JSON text ('{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>': Extra data: line 1 column 4 (char 3))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like ': Unterminated string starting at: line 1 column 10 (char 9)); [1] f: arguments are not valid JSON text ('{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>': Extra data: line 1 column 4 (char 3))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\":': Expecting value: line 1 column 9 (char 8)); [1] f: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like ': Unterminated string starting at: line 1 column 10 (char 9)); [1] f: arguments are not valid JSON text ('{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>': Extra data: line 1 column 4 (char 3))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like ': Unterminated string starting at: line 1 column 10 (char 9)); [1] f: arguments are not valid JSON text ('{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>': Extra data: line 1 column 4 (char 3))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] echo: arguments are not a JSON object; not validated; [1] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] echo: arguments are not a JSON object; not validated; [1] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] echo: arguments are not a JSON object; not validated; [1] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] echo: arguments are not a JSON object; not validated; [1] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] echo: arguments are not a JSON object; not validated; [1] 'f' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] echo: arguments are not a JSON object; not validated; [1] 'f' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "f",
"arguments_raw": ""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "f",
"arguments_raw": ""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Tokens like "
},
{
"name": "f",
"arguments_raw": "{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Tokens like"
},
{
"name": "f",
"arguments_raw": "{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Tokens like "
},
{
"name": "f",
"arguments_raw": "{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\":"
},
{
"name": "f",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Tokens like "
},
{
"name": "f",
"arguments_raw": "{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Tokens like "
},
{
"name": "f",
"arguments_raw": "{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-truncated-mid-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Zür'"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": ""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zür"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-multi-turn-after-tool-result",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['search'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['search'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['search'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['search'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['search'] | rand:1:8 (fail): tool_calls: one [], got ['search'] | rand:2:8 (fail): tool_calls: one [], got ['search'] | rand:3:8 (fail): tool_calls: one [], got ['search'] | rand:4:8 (fail): tool_calls: one [], got ['search'] | rand:5:8 (fail): tool_calls: one [], got ['search']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"query\": \"café Zürich\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v31-nested-json",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['create_event'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['create_event'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['create_event'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['create_event'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..."
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['create_event'] | rand:1:8 (fail): tool_calls: one [], got ['create_event'] | rand:2:8 (fail): tool_calls: one [], got ['create_event'] | rand:3:8 (fail): tool_calls: one [], got ['create_event'] | rand:4:8 (fail): tool_calls: one [], got ['create_event'] | rand:5:8 (fail): tool_calls: one [], got ['create_event']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] create_event: arguments are not valid JSON text ('{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam...: Expecting value: line 1 column 193 (char 192))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] create_event: arguments are not valid JSON text ('{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam...: Unterminated string starting at: line 1 column 195 (char 194))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] create_event: arguments are not valid JSON text ('{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam...: Expecting value: line 1 column 193 (char 192))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] create_event: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] create_event: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] create_event: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v31",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.1",
"revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"requested_mode": "auto",
"resolved_revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.1",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0"
},
"prompt_tail": "<｜begin▁of▁sentence｜><｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\":"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\":"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v32-single-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"city\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-parallel-two-calls",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": ": [\"query\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": ": [\"query\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-text-before-call",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": ": [\"query\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v32-empty-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "erties\": {}}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-numeric-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": " [\"x\", \"y\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-unicode-emoji",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"text\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-marker-in-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"text\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-truncated-mid-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: $: 'city' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"text\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v32-reasoning-single-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "d\": [\"city\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-reasoning-parallel-text-before",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"query\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"special": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"token": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:1:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:2:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:3:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:4:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:5:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v32-reasoning-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"query\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-string-false-json",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "e\", \"when\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-unescaped-string-value",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": " \"content\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-truncated-in-reasoning",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": "\": [\"query\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v32-truncated-second-parallel-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v32",
"reasoning_parser": "deepseek_v3",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V3\\.2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V3.2",
"revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"requested_mode": "auto",
"resolved_revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6",
"mode": "deepseek_v32",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V3.2",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v32",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v32 encoder (vllm.tokenizers.deepseek_v32_encoding)",
"sha256": "1bade67606d54952ae652a4f71f44065041ef0842ad1b5420ed82f8159502bc8"
},
"prompt_tail": " \"content\"]}}\n\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v4-single-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-parallel-two-calls",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-text-before-call",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v4-empty-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-numeric-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-unicode-emoji",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-marker-in-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-truncated-mid-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: $: 'city' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v4-multi-turn-after-tool-result",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-reasoning-single-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-reasoning-parallel-text-before",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"special": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"token": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:1:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:2:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:3:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:4:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:5:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v4-reasoning-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-string-false-json",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-unescaped-string-value",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-truncated-in-reasoning",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v4-truncated-second-parallel-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v4",
"reasoning_parser": "deepseek_v4",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4-Flash",
"revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"requested_mode": "auto",
"resolved_revision": "60d8d70770c6776ff598c94bb586a859a38244f1",
"mode": "deepseek_v4",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v4",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v4 encoder (vllm.tokenizers.deepseek_v4_encoding)",
"sha256": "bfbb51e563b89c7d3581864cd0ea4d975b96e14fed034a222caf9d50c549b99d"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v41-single-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-parallel-two-calls",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-text-before-call",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current conditions in London.\n\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v41-empty-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-numeric-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-unicode-emoji",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-marker-in-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-truncated-mid-arguments",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got '{\"city\": \"'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: $: 'city' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v41-multi-turn-after-tool-result",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-reasoning-single-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-reasoning-parallel-text-before",
"family": "deepseek",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"special": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"token": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:1:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:2:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:3:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:4:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
},
"rand:5:8": {
"content": "I'll look both up.\n\n",
"reasoning_content": "Two independent lookups: weather and search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v41-reasoning-no-call",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-string-false-json",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-unescaped-string-value",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-truncated-in-reasoning",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "hemas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-truncated-second-parallel-call",
"family": "deepseek",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] search: $: 'query' is a required property"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "deepseek/v41-long-arguments",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "deepseek/v41-whitespace-string-value",
"family": "deepseek",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "deepseek_v41",
"reasoning_parser": "deepseek_v41",
"parser_source": "docs/formats/deepseek.md (vLLM v0.30.0 row)",
"model_rule": "DeepSeek-V4\\.1",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "deepseek-ai/DeepSeek-V4.1-Flash",
"revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"requested_mode": "auto",
"resolved_revision": "dba1be0a40aa45a94ad051997016db3960a90277",
"mode": "deepseek_v41",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "deepseek-ai/DeepSeek-V4.1-Flash",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<｜Assistant｜>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "deepseek_v41",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=deepseek_v41 encoder (vllm.tokenizers.deepseek_v41_encoding)",
"sha256": "081382cb953a1c8041fdfc82a6001cf0c932553171436625c89fe5b9fbcda464"
},
"prompt_tail": "emas to invoke tool calls.\n<｜User｜>Use the tools to answer.<｜Assistant｜>",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/single-call",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/single-call-thinking",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/no-call",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/no-call-thinking",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/text-after-call",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got None"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/text-after-call-thinking",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected 'Let me check the current conditions in London.', got None"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check the current conditions in London.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current conditions in London.",
"reasoning_content": "I need live weather data for London.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/empty-arguments",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/empty-arguments-thinking",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/bug-ollama-18390-key-with-spaces",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": 0}]]}}}', got '{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_workflow",
"arguments_raw": "{\"name\": \"Demo\", \"connections\": {\"Basic LLM Chain\": {\"main\": [[{\"node\": \"X\", \"type\": \"main\", \"index\": \"0\"}]]}}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/nested-objects-and-arrays",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"no..."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": \"false\"}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": \"true\", \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": null, \"reminders\": [[\"10\", \"email\"], [\"1\", \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/edge-values-key-with-space",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "record",
"arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": null, \"obj\": {\"a\": [\"1\", \"t\"], \"B\": \"1\"}, \"s\": \"x,y} \\\"q\\\"\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/dictsort-case-insensitive-order",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/marker-in-arguments",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected None, got ' and close thoughts with .<|\"|>}'; tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got ' and close thoughts with .<|\"|>}'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content contains '<|\"|>'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"special": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"token": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": " and close thoughts with .<|\"|>}",
"reasoning_content": null,
"tool_calls": [
{
"name": "echo",
"arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/unicode-emoji",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/numeric-arguments",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/string-with-newlines-and-backslashes",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/json-text-in-string",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/array-of-strings",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/long-arguments",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/array-of-objects-with-braces",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/many-strings-then-string-array",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-no-tool-calls",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-single-call",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-multiple-arguments",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-text-before-call",
"family": "gemma4",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the weather for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the weather for you. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Paris\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/vllm-two-calls",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-nested-arguments",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-number-and-boolean",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-hyphenated-function-name",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-dotted-function-name",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-no-arguments",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-reasoning-thought-prefix",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/vllm-reasoning-thought-prefix-multiline",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/sglang-text-before-call",
"family": "gemma4",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Some text before",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"special": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"token": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Some text before ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/sglang-text-around-call",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected 'Some text before after', got 'Some text before'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'Some text before', got 'Some text before after'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Some text before",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"special": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"token": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Some text before after",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/sglang-nested-array-with-spaces",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"New York\", \"nested\": [1, 2, {\"inner\": \"val\"}]}', got '{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"New York\", \"nested\": [\"1\", \"2\", {\"inner\": \"val\"}]}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/sglang-nested-object",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"location\": \"Tokyo\", \"details\": {\"temp\": 25, \"unit\": \"celsius\"}}', got '{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"location\": \"Tokyo\", \"details\": {\"temp\": \"25\", \"unit\": \"celsius\"}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/vllm-malformed-no-brace",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['bad_func no brace'] for The call has no '{...}' argument object and names no offered tool; it is not a valid call."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] 'bad_func no brace' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "bad_func no brace",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/multi-turn-final-answer",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/multi-turn-second-call",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got '{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"filters\": {\"open_now\": \"true\"}, \"query\": \"café Zürich\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/parallel-two-calls-thinking",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\\\"best\\\\\"\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "I should call the tools.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/parallel-three-calls",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/parallel-mixed-with-empty",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/truncated-mid-string",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/truncated-in-name",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] 'get' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/truncated-before-close",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gemma4/truncated-in-reasoning",
"family": "gemma4",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "T<|\"|>}}\n<|turn>user\nUse the tools to answer.\n<|turn>model\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "gemma4/truncated-second-parallel-call",
"family": "gemma4",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "gemma4",
"reasoning_parser": "gemma4",
"parser_source": "docs/formats/gemma4.md (vLLM v0.30.0 row)",
"model_rule": "gemma-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "google/gemma-4-31B-it",
"revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"requested_mode": "auto",
"resolved_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
"mode": "hf",
"tokenizer_class": "transformers.models.gemma.tokenization_gemma.GemmaTokenizer"
},
"model": "google/gemma-4-31B-it",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|turn>model\n<|channel>thought\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gemma4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
},
"prompt_tail": "n>user\nUse the tools to answer.\n<|turn>model\n<|channel>thought\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}"
},
{
"name": "search",
"arguments_raw": "{\"filters\": {\"max\": \"3\", \"tags\": [\"a\", \"b\"]}, \"query\": \"\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-single-call",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got 'The user wants the current weather in Paris, in Celsius.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "The user wants the current weather in Paris, in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-parallel-same-tool",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nTwo cities, so two independent weather lookups.', got 'Two cities, so two independent weather lookups.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nTwo cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Two cities, so two independent weather lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-parallel-mixed-nested",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nCheck the weather, then search for cafés.', got 'Check the weather, then search for cafés.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nCheck the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Check the weather, then search for cafés.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-content-before-call",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\n\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): nonstream '\\nNeed live data, so call the tool.', got 'Need live data, so call the tool.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I'll look that up for you.",
"reasoning_content": "\nNeed live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"special": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"token": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "\n\nI'll look that up for you.\n",
"reasoning_content": "Need live data, so call the tool.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-no-call",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\n\\nParis is the capital of France.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream '\\nParis is the capital of France.', got '\\n\\nParis is the capital of France.'; reasoning_content (whitespace only): nonstream '\\nA simple fact; no tool is needed.', got 'A simple fact; no tool is needed.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "\nParis is the capital of France.",
"reasoning_content": "\nA simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"special": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"token": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "\n\nParis is the capital of France.",
"reasoning_content": "A simple fact; no tool is needed.",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-nested-object",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nApply the dark theme and disable SMS.', got 'Apply the dark theme and disable SMS.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nApply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Apply the dark theme and disable SMS.",
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-unicode-emoji",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\""
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream \"\\nSearch with the user's exact wording.\", got \"Search with the user's exact wording.\""
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nSearch with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Search with the user's exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-empty-arguments",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nJust fetch the time.', got 'Just fetch the time.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nJust fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Just fetch the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-marker-in-arguments",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected None, got '; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected None, got '; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected None, got '; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected None, got '\\n; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected None, got '\\n\\n; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .\\n'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got '\\n; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got '\\n\\n; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content (whitespace only): nonstream '\\nWrite the format note to the docs file.', got 'Write the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "soft_pass",
"detail": "vs one: rand:1:8 (soft_pass): content (whitespace only): one '; keys go in .\\n', got '\\n; keys go in .\\n' | rand:3:8 (soft_pass): content (whitespace only): one '; keys go in .\\n', got '\\n\\n; keys go in .\\n'"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] write_file: $: 'content' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nWrite the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"special": {
"content": "; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"token": {
"content": "; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:1:8": {
"content": "\n; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:2:8": {
"content": "; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:3:8": {
"content": "\n\n; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:4:8": {
"content": "; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:5:8": {
"content": "; keys go in .\n",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-schema-coercion",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nThree second timer labelled 3.', got 'Three second timer labelled 3.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nThree second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Three second timer labelled 3.",
"tool_calls": [
{
"name": "set_timer",
"arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-thinking-disabled",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "tool_call><|user|>\nUse the tools to answer./nothink<|assistant|>\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm45-call-order-vs-tool-order",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): nonstream '\\nWeather, then search, then weather again.', got 'Weather, then search, then weather again.'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nWeather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Weather, then search, then weather again.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Oslo\"}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Oslo museums\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Bergen\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm47-single-call",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-parallel-same-tool",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-content-before-call",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-no-call",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-nested-object",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-unicode-emoji",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-empty-arguments",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-marker-in-arguments",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and '"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ': Unterminated string starting at: line 1 column 39 (char 38))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] write_file: $: 'content' is a required property"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"special": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"token": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:1:8": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:2:8": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:3:8": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:4:8": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
},
"rand:5:8": {
"content": "; keys go in .",
"reasoning_content": "Write the format note to the docs file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm47-typed-values",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-number-looking-string",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-schema-coercion",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-string-escapes",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-significant-whitespace",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-long-arguments",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-json-looking-string",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-thinking-disabled",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-call-order-vs-tool-order",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-multi-turn-second-call",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm53-single-call",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-5.3",
"revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"requested_mode": "auto",
"resolved_revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-5.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm_moe_dsa",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm53-parallel-same-tool",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-5.3",
"revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"requested_mode": "auto",
"resolved_revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-5.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm_moe_dsa",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm53-no-call",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-5.3",
"revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"requested_mode": "auto",
"resolved_revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-5.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm_moe_dsa",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm53-schema-coercion",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-5.3",
"revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"requested_mode": "auto",
"resolved_revision": "aca966e4e02791568aa6a4ced368624b3d897f42",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-5.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm_moe_dsa",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm45-vllm-newline-format-content-before",
"family": "glm",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \""
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "tool_call><|user|>\nUse the tools to answer./nothink<|assistant|>\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I'll check it.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"special": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"token": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "I'll check it. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_current_weather",
"arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-vllm-newline-format-two-calls",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "tool_call><|user|>\nUse the tools to answer./nothink<|assistant|>\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm45-vllm-coerces-schema-types",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "tool_call><|user|>\nUse the tools to answer./nothink<|assistant|>\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm45-vllm-zero-arg-newline",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "tool_call><|user|>\nUse the tools to answer./nothink<|assistant|>\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm45-vllm-with-think-tags",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm45-ollama-content-after-call",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected 'after tool', got None"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got 'after tool'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "ue-2}\n...\n<|user|>\nUse the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": "after tool",
"reasoning_content": "thinking",
"tool_calls": [
{
"name": "test",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm47-vllm-whitespace-preserved-in-value",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-vllm-content-before-no-space",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-vllm-whitespace-only-content",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-vllm-args-with-newlines",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-vllm-reasoning-without-open-tag",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-bug-three-calls-same-tool",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-truncated-inside-arg-value",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm47-truncated-before-close-call",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm45-truncated-after-arg-key",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm45",
"reasoning_parser": "glm45",
"parser_source": "GLM-4.5/4.6 model cards: --tool-call-parser glm45 --reasoning-parser glm45",
"model_rule": "GLM-4\\.[56]",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.5",
"revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"requested_mode": "auto",
"resolved_revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.5",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3"
},
"prompt_tail": "tool_call><|user|>\nUse the tools to answer./nothink<|assistant|>\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "glm/glm47-truncated-in-reasoning",
"family": "glm",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "}...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "glm/glm47-missing-last-close-arg-value",
"family": "glm",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius': Unterminated string starting at: line 1 column 28 (char 27))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "glm47",
"reasoning_parser": "glm47",
"parser_source": "docs/formats/glm.md (vLLM v0.30.0 row)",
"model_rule": "GLM-(4\\.7|5)",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "zai-org/GLM-4.7",
"revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"requested_mode": "auto",
"resolved_revision": "602d01efcdd332c5238ca4bcede555defbe83eb7",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "zai-org/GLM-4.7",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|assistant|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "glm4_moe",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375"
},
"prompt_tail": "...<|user|>Use the tools to answer.<|assistant|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gpt-oss/vllm-channel-first-recipient",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-channel-first-empty-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-split-deltas-constrain-nospace",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-call-on-analysis-channel",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-preamble-call-end-terminated",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-sequential-calls",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-call-then-final",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-truncated-final",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-interrupted-final-only",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/vllm-malformed-headers",
"family": "gpt-oss",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'thinking\\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}', got 'thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "thinking\n<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"special": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"token": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "thinking<|start|>assistant<|channel|>final {\"answer\": \"hi\"}",
"reasoning_content": "thinking",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "gpt-oss/llamacpp-commentary-content-only",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/llamacpp-recipient-in-channel-analysis",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/llamacpp-recipient-in-channel-commentary",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/llamacpp-recipient-in-role-constrain",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/llamacpp-edit-code-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/llamacpp-builtin-python-recipient",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/llamacpp-stray-commentary-header",
"family": "gpt-oss",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['assistant<|channel|>analysis'] for Stray '<|channel|>commentary to=assistant' header before the analysis channel. llama.cpp recovers reasoning 'I'm\\nthinking' and content 'Hello, world!\\nWhat's up?'; openai-harmony's non-strict parser keeps the content but drops the reasoning. Either way there is no tool call and the turn must not error."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].name contains '<|channel|>'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] assistant<|channel|>analysis: arguments are not valid JSON text (\"I'm\\nthinking\": Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] 'assistant<|channel|>analysis' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"special": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"token": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"rand:1:8": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"rand:2:8": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"rand:3:8": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"rand:4:8": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
},
"rand:5:8": {
"content": "Hello, world!\nWhat's up?",
"reasoning_content": null,
"tool_calls": [
{
"name": "assistant<|channel|>analysis",
"arguments_raw": "I'm\nthinking"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gpt-oss/harmony-test-channel-first-constrain-adjacent",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-test-role-recipient-constrain-adjacent",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-spec-receiving-tool-calls",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-spec-preamble-action-plan",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/bug-unknown-channel-comment",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/bug-garbled-channel-question-marks",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/bug-garbled-channel-commentary-question",
"family": "gpt-oss",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "outcome exception 'HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")'; accepted: ['content_passthrough', 'no_tool_calls']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": "HarmonyError: unexpected tokens remaining in message header: Some(\"need temperature difference. Also need outside temperature. Get\")"
}
}
}
},
{
"fixture_id": "gpt-oss/harmony-single-call",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-single-call-no-reasoning",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-compact-json",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-unicode-emoji",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-nested-json",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-empty-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-marker-in-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-marker-in-reasoning",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-preamble-then-call",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-preamble-no-reasoning",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-no-call-final",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-final-only",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-final-json-lookalike",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-string-escapes",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-numeric-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-long-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-multiline-reasoning",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-content-type-json-text",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-no-content-type",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-truncated-in-arguments",
"family": "gpt-oss",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Output stopped (max_tokens) inside the tool-call JSON; there is no complete call."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Paris\", \"unit': Unterminated string starting at: line 1 column 19 (char 18))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "gpt-oss/harmony-truncated-in-reasoning",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/harmony-truncated-in-header",
"family": "gpt-oss",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content contains '<|start|>'; content contains '<|channel|>'; content contains '<|constrain|>'"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"special": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"token": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json",
"reasoning_content": "The user asks for the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "gpt-oss/hf-template-single-call",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/hf-template-unicode-nested",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/hf-template-empty-arguments",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/hf-template-no-reasoning",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "gpt-oss/hf-template-final-answer",
"family": "gpt-oss",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "openai",
"reasoning_parser": "openai_gptoss",
"parser_source": "vLLM gpt-oss recipe; HarmonyParser (is_harmony)",
"model_rule": "gpt-oss",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "openai/gpt-oss-20b",
"revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"requested_mode": "auto",
"resolved_revision": "6cee5e81ee83917806bbde320786a8fb61efebee",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "openai/gpt-oss-20b",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start|>assistant",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "gpt_oss",
"is_harmony": true,
"template": {
"source": "openai_harmony via vLLM OnlineRenderer._make_request_with_harmony (no Jinja template)",
"openai_harmony": "0.0.8"
},
"prompt_tail": "<|end|><|start|>user<|message|>Use the tools to answer.<|end|><|start|>assistant",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k2i-vllm-single-call-content-before",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected \"I'll check. \", got \" I'll check. \""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"special": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"token": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": " I'll check. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-vllm-three-tool-calls",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Multiple tasks. ', got ' Multiple tasks. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"special": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"token": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": " Multiple tasks. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"New York\"}"
},
{
"name": "get_news",
"arguments_raw": "{\"topic\": \"technology\"}"
},
{
"name": "send_email",
"arguments_raw": "{\"to\": \"user@example.com\", \"subject\": \"Daily Update\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-vllm-multiline-json",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Formatted. ', got ' Formatted. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"special": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"token": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"rand:1:8": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"rand:2:8": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"rand:3:8": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"rand:4:8": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
},
"rand:5:8": {
"content": " Formatted. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "process_data",
"arguments_raw": "{\n \"name\": \"test\",\n \"value\": 123\n}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-vllm-id-without-functions-prefix",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'No prefix. ', got ' No prefix. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"special": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"token": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": " No prefix. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-vllm-noise-between-markers",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"special": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"token": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "test",
"arguments_raw": "{\"k\": \"v\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-vllm-empty-tool-section",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Reasoning. ', got ' Reasoning. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": " Reasoning. ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-vllm-content-after-tool-section",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected 'Before. After tools.', got ' Before. '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"special": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"token": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": " Before. ",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k3-vllm-no-call-thinking-disabled",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "ep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-vllm-call-thinking-disabled",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "ep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-vllm-truncated-tools",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "ep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-vllm-reasoning-prefix-consumed",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-bug-truncated-reasoning-recorded",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected None, got 'The user asks to reason step by step about a math problem. I only need '; reasoning_content: expected 'The user asks to reason step by step about a math problem. I only need ', got None"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'The user asks to reason step by step about a math problem. I only need ', got None; reasoning_content: nonstream None, got 'The user asks to reason step by step about a math problem. I only need '"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "The user asks to reason step by step about a math problem. I only need ",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "The user asks to reason step by step about a math problem. I only need ",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k3-bug-response-only-completion",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected 'Hello world', got None; reasoning_content: expected None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'Hello world', got None; reasoning_content: nonstream None, got '<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "reasoning_content contains '<|open|>'; reasoning_content contains '<|close|>'; reasoning_content contains '<|sep|>'"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Hello world",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "<|open|>response<|sep|>Hello world<|close|>response<|sep|><|close|>message<|sep|>",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-single-call",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris in Celsius.', got ' The user wants the weather in Paris in Celsius.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris in Celsius.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\", \"unit\": \"celsius\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-parallel-same-tool",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Two cities, two lookups.', got ' Two cities, two lookups.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Two cities, two lookups.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Paris\"} "
},
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Tokyo\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-parallel-three-tools",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Weather, time and a search.', got ' Weather, time and a search.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Zürich\"}"
},
{
"name": "get_time",
"arguments_raw": "{}"
},
{
"name": "search_web",
"arguments_raw": "{\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Weather, time and a search.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Zürich\"} "
},
{
"name": "get_time",
"arguments_raw": " {} "
},
{
"name": "search_web",
"arguments_raw": " {\"query\": \"cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 3}} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-content-before-call",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you. '; reasoning_content (whitespace only): expected 'Live data needed.', got ' Live data needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"special": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"token": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": " Let me check that for you. ",
"reasoning_content": " Live data needed.",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Berlin\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-no-call",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool needed.', got ' A simple fact; no tool needed.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"special": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"token": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": " Paris is the capital of France.",
"reasoning_content": " A simple fact; no tool needed.",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-nested-json",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Create the event with a structured location.', got ' Create the event with a structured location.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Create the event with a structured location.",
"tool_calls": [
{
"name": "create_event",
"arguments_raw": " {\"title\": \"Offsite\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"location\": {\"name\": \"HQ\", \"geo\": {\"lat\": 47.3769, \"lon\": 8.5417}, \"floors\": [1, 2]}} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-unicode-emoji",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Search with the exact wording.', got ' Search with the exact wording.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Search with the exact wording.",
"tool_calls": [
{
"name": "search_web",
"arguments_raw": " {\"query\": \"Zürich café ☕ 東京の天気 🌸\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-empty-arguments",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Just get the time.', got ' Just get the time.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Just get the time.",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": " {} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-string-escapes",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Save the note verbatim.', got ' Save the note verbatim.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Save the note verbatim.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" \\\\ end\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-marker-in-arguments",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected None, got ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w...; reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected None, got ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w...; reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Document the tool-call markers.', got ' Document the tool-call markers.'; tool_calls[0].arguments: expected '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|>; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w...; tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w...; tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}', got ' {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasonin..."
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): content: one ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w..., got None; tool_calls: one [], got ['write_file'] | rand:1:8 (fail): content: one ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w..., got None; tool_calls: one [], got ['write_file'] | rand:2:8 (fail): content: one ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w..., got None; tool_calls: one [], got ['write_file'] | rand:3:8 (fail): content: one ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w..., got None; tool_calls: one [], got ['write_file'] | rand:4:8 (fail): content: one ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w..., got None; tool_calls: one [], got ['write_file'] | rand:5:8 (fail): content: one ' <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens w..., got None; tool_calls: one [], got ['write_file']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "content contains '<|tool_calls_section_begin|>'; content contains '<|tool_calls_section_end|>'; content contains '<|tool_call_begin|>'; content contains '<|tool_call_end|>'; content contains '<|tool_call_argument_begin|>'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "content contains '<|tool_calls_section_begin|>'; content contains '<|tool_calls_section_end|>'; content contains '<|tool_call_begin|>'; content contains '<|tool_call_end|>'; content contains '<|tool_call_argument_begin|>'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} <|tool_call_end|> <|tool_calls_section_end|>",
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [],
"exception": null
},
"special": {
"content": " <|tool_calls_section_begin|> <|tool_call_begin|> functions.write_file:0 <|tool_call_argument_begin|> {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} <|tool_call_end|> <|tool_calls_section_end|>",
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Document the tool-call markers.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"docs/kimi.md\", \"content\": \"A section opens with <|tool_calls_section_begin|> and each call with <|tool_call_begin|> ; ends reasoning.\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-marker-terminator-in-string",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "reasoning_content (whitespace only): expected 'Write the literal end marker into the file.', got ' Write the literal end marker into the file.'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at', got ' {\"path\": \"end.txt\", \"content\": \"stop at '"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at': Unterminated string starting at: line 1 column 32 (char 31))"
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text (' {\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 33 (char 32))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Write the literal end marker into the file.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": " {\"path\": \"end.txt\", \"content\": \"stop at "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k26-empty-think",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k26-thinking-disabled-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": " tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k26-multi-turn-second-call",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Now convert it.', got ' Now convert it.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " Now convert it.",
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": " {\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"} "
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-single-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k2i-content-before-call",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check that for you.', got ' Let me check that for you.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"special": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"token": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": " Let me check that for you.",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Berlin\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k2i-no-call",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Paris is the capital of France.', got ' Paris is the capital of France.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": null,
"parser_source": "Kimi-K2-Instruct model card: non-thinking; --tool-call-parser kimi_k2",
"model_rule": "Kimi-K2-Instruct",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "moonshotai/Kimi-K2-Instruct-0905",
"revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"requested_mode": "auto",
"resolved_revision": "ac6c49f04883bd0a0598b790693a72061c676629",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_hyphen_Instruct_hyphen_0905.ac6c49f04883bd0a0598b790693a72061c676629.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2-Instruct-0905",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k2",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b"
},
"prompt_tail": "middle|>Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": " Paris is the capital of France.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k3-single-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-parallel-same-tool",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-content-before-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-no-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-nested-json",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-typed-values",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-empty-arguments",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-string-escapes",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-control-marker-text-in-value",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Write the literal XTML marker text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k3-xtml-attr-escaping",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-long-arguments",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-empty-think",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-thinking-disabled-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "ep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-thinking-disabled-no-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "ep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k3-multi-turn-second-call",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k26-truncated-inside-arguments",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Madrid\", \"unit\":', got ' {\"city\": \"Madrid\", \"unit\":'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))"
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text (' {\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 28 (char 27))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": " tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": " {\"city\": \"Madrid\", \"unit\":"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k3-truncated-inside-argument",
"family": "kimi",
"status": "pass",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "ep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
"generation_prompt_match": true
}
},
{
"fixture_id": "kimi/k26-truncated-in-reasoning",
"family": "kimi",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'The user wants the weather in Paris', got ' The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k2",
"reasoning_parser": "kimi_k2",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row; thinking K2.x)",
"model_rule": "Kimi-K2",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K2.6",
"revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"requested_mode": "auto",
"resolved_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb",
"mode": "hf",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K2_dot_6.7eb5002f6aadc958aed6a9177b7ed26bb94011bb.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K2.6",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_assistant|>assistant<|im_middle|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k25",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f"
},
"prompt_tail": ">Use the tools to answer.<|im_end|><|im_assistant|>assistant<|im_middle|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": " The user wants the weather in Paris",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "kimi/k3-truncated-in-reasoning",
"family": "kimi",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected None, got 'The user wants the weather in Paris'; reasoning_content: expected 'The user wants the weather in Paris', got None"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'The user wants the weather in Paris', got None; reasoning_content: nonstream None, got 'The user wants the weather in Paris'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "kimi_k3",
"reasoning_parser": "kimi_k3",
"parser_source": "docs/formats/kimi.md (vLLM v0.30.0 row)",
"model_rule": "Kimi-K3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "moonshotai/Kimi-K3",
"revision": "f831ab66814297da540d832a5235f8e904f29d06",
"requested_mode": "auto",
"resolved_revision": "f831ab66814297da540d832a5235f8e904f29d06",
"mode": "kimi_k3",
"tokenizer_class": "transformers_modules.moonshotai.Kimi_hyphen_K3.f831ab66814297da540d832a5235f8e904f29d06.tokenization_kimi.TikTokenTokenizer"
},
"model": "moonshotai/Kimi-K3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "kimi_k3",
"is_harmony": false,
"template": {
"source": "vLLM tokenizer_mode=kimi_k3 (tokenizer.apply_chat_template, repo encoder)",
"sha256": null
},
"prompt_tail": "<|sep|><|end_of_msg|><|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "The user wants the weather in Paris",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "The user wants the weather in Paris",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-vllm-parameters-key",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['searchTool'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['searchTool'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['searchTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['searchTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['searchTool'] | rand:1:8 (fail): tool_calls: one [], got ['searchTool'] | rand:2:8 (fail): tool_calls: one [], got ['searchTool'] | rand:3:8 (fail): tool_calls: one [], got ['searchTool'] | rand:4:8 (fail): tool_calls: one [], got ['searchTool'] | rand:5:8 (fail): tool_calls: one [], got ['searchTool']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test query\", \"limit\": 10}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-vllm-arguments-key",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['searchTool'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['searchTool'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['searchTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['searchTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['searchTool'] | rand:1:8 (fail): tool_calls: one [], got ['searchTool'] | rand:2:8 (fail): tool_calls: one [], got ['searchTool'] | rand:3:8 (fail): tool_calls: one [], got ['searchTool'] | rand:4:8 (fail): tool_calls: one [], got ['searchTool'] | rand:5:8 (fail): tool_calls: one [], got ['searchTool']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-vllm-deeply-nested",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['complexTool'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['complexTool'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['complexTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['complexTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['complexTool'] | rand:1:8 (fail): tool_calls: one [], got ['complexTool'] | rand:2:8 (fail): tool_calls: one [], got ['complexTool'] | rand:3:8 (fail): tool_calls: one [], got ['complexTool'] | rand:4:8 (fail): tool_calls: one [], got ['complexTool'] | rand:5:8 (fail): tool_calls: one [], got ['complexTool']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "complexTool",
"arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-vllm-brackets-in-strings",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['searchTool'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['searchTool'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['searchTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['searchTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['searchTool'] | rand:1:8 (fail): tool_calls: one [], got ['searchTool'] | rand:2:8 (fail): tool_calls: one [], got ['searchTool'] | rand:3:8 (fail): tool_calls: one [], got ['searchTool'] | rand:4:8 (fail): tool_calls: one [], got ['searchTool'] | rand:5:8 (fail): tool_calls: one [], got ['searchTool']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "searchTool",
"arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-vllm-escaped-quotes",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['parserTool'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['parserTool'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['parserTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['parserTool'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['parserTool'] | rand:1:8 (fail): tool_calls: one [], got ['parserTool'] | rand:2:8 (fail): tool_calls: one [], got ['parserTool'] | rand:3:8 (fail): tool_calls: one [], got ['parserTool'] | rand:4:8 (fail): tool_calls: one [], got ['parserTool'] | rand:5:8 (fail): tool_calls: one [], got ['parserTool']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "parserTool",
"arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-vllm-json-without-name-is-content",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '{\"parameters\": {}}', got None"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "{\"parameters\": {}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-sglang-python-tag-call",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Beijing\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-sglang-text-before-python-tag",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected 'Let me check. ', got None"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected 'Let me check. ', got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream None, got 'Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "Let me check. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l4-vllm-pythonic-no-call",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-vllm-pythonic-more-types",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-vllm-pythonic-parameterless",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-vllm-pythonic-escaped-strings",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-vllm-pythonic-parallel-no-spaces",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-vllm-pythonic-python-start-end",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l3-bug-leading-json-object-in-content",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '{\"a\": 1} is a dict', got None"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "{\"a\": 1} is a dict",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-bug-empty-json-object-in-content",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '{}', got None"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "{}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-bug-whole-call-single-delta",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Tokyo\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l4-bug-leading-underscore-identifier",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '[search(_limit=5)]', got None; tool_calls: nonstream [], got ['search']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "[search(_limit=5)]",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search",
"arguments_raw": "{\"_limit\": 5}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-single-call",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_weather'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_weather'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-nested-json",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['update_settings'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['update_settings'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['update_settings'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['update_settings'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['update_settings'] | rand:1:8 (fail): tool_calls: one [], got ['update_settings'] | rand:2:8 (fail): tool_calls: one [], got ['update_settings'] | rand:3:8 (fail): tool_calls: one [], got ['update_settings'] | rand:4:8 (fail): tool_calls: one [], got ['update_settings'] | rand:5:8 (fail): tool_calls: one [], got ['update_settings']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "update_settings",
"arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-unicode-emoji",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['search_web'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['search_web'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['search_web'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['search_web'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['search_web'] | rand:1:8 (fail): tool_calls: one [], got ['search_web'] | rand:2:8 (fail): tool_calls: one [], got ['search_web'] | rand:3:8 (fail): tool_calls: one [], got ['search_web'] | rand:4:8 (fail): tool_calls: one [], got ['search_web'] | rand:5:8 (fail): tool_calls: one [], got ['search_web']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "search_web",
"arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-empty-arguments",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['get_time'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['get_time'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{}', got ''"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['get_time'] | rand:1:8 (fail): tool_calls: one [], got ['get_time'] | rand:2:8 (fail): tool_calls: one [], got ['get_time'] | rand:3:8 (fail): tool_calls: one [], got ['get_time'] | rand:4:8 (fail): tool_calls: one [], got ['get_time'] | rand:5:8 (fail): tool_calls: one [], got ['get_time']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_time: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_time",
"arguments_raw": ""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-marker-in-arguments",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: special (fail): tool_calls: one [], got ['write_file'] | token (fail): tool_calls: one [], got ['write_file'] | rand:1:8 (fail): tool_calls: one [], got ['write_file'] | rand:2:8 (fail): tool_calls: one [], got ['write_file'] | rand:3:8 (fail): tool_calls: one [], got ['write_file'] | rand:4:8 (fail): tool_calls: one [], got ['write_file'] | rand:5:8 (fail): tool_calls: one [], got ['write_file']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like ipython.\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-string-escapes",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['write_file'] | rand:1:8 (fail): tool_calls: one [], got ['write_file'] | rand:2:8 (fail): tool_calls: one [], got ['write_file'] | rand:3:8 (fail): tool_calls: one [], got ['write_file'] | rand:4:8 (fail): tool_calls: one [], got ['write_file'] | rand:5:8 (fail): tool_calls: one [], got ['write_file']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-typed-values",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['create_event'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['create_event'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['create_event'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['create_event'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['create_event'] | rand:1:8 (fail): tool_calls: one [], got ['create_event'] | rand:2:8 (fail): tool_calls: one [], got ['create_event'] | rand:3:8 (fail): tool_calls: one [], got ['create_event'] | rand:4:8 (fail): tool_calls: one [], got ['create_event'] | rand:5:8 (fail): tool_calls: one [], got ['create_event']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-long-arguments",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['write_file'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['write_file'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['write_file'] | rand:1:8 (fail): tool_calls: one [], got ['write_file'] | rand:2:8 (fail): tool_calls: one [], got ['write_file'] | rand:3:8 (fail): tool_calls: one [], got ['write_file'] | rand:4:8 (fail): tool_calls: one [], got ['write_file'] | rand:5:8 (fail): tool_calls: one [], got ['write_file']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-no-call",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l3-json-answer-not-a-call",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '{\"capital\": \"Paris\", \"country\": \"France\"}', got None"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "{\"capital\": \"Paris\", \"country\": \"France\"}",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-multi-turn-second-call",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['convert_units'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['convert_units'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['convert_units'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['convert_units'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['convert_units'] | rand:1:8 (fail): tool_calls: one [], got ['convert_units'] | rand:2:8 (fail): tool_calls: one [], got ['convert_units'] | rand:3:8 (fail): tool_calls: one [], got ['convert_units'] | rand:4:8 (fail): tool_calls: one [], got ['convert_units'] | rand:5:8 (fail): tool_calls: one [], got ['convert_units']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "convert_units",
"arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-meta-python-tag-type-function",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['trending_songs'], got []"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['trending_songs'], got []"
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream ['trending_songs'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['trending_songs'], got []"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['trending_songs'] | rand:1:8 (fail): tool_calls: one [], got ['trending_songs'] | rand:2:8 (fail): tool_calls: one [], got ['trending_songs'] | rand:3:8 (fail): tool_calls: one [], got ['trending_songs'] | rand:4:8 (fail): tool_calls: one [], got ['trending_songs'] | rand:5:8 (fail): tool_calls: one [], got ['trending_songs']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "trending_songs",
"arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l4-meta-pythonic-parallel",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-meta-pythonic-int-and-string",
"family": "llama",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "llama/l4-truncated-missing-close-bracket",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 2 tool call(s) ['get_weather', 'get_weather'] for The pythonic call list is cut before its final string, call and list close."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None; tool_calls: nonstream [], got ['get_weather', 'get_weather']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not valid JSON text ('{\"city\": \"Seattle': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[1] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama4_pythonic",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-4",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
"revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"requested_mode": "auto",
"resolved_revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|header_start|>assistant<|header_end|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama4",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd"
},
"prompt_tail": "_end|>\n\nUse the tools to answer.<|eot|><|header_start|>assistant<|header_end|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Francisco\"}"
},
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Seattle"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "llama/l3-truncated-inside-parameters",
"family": "llama",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: token (fail): tool_calls: one [], got ['create_event'] | rand:1:8 (fail): tool_calls: one [], got ['create_event'] | rand:2:8 (fail): tool_calls: one [], got ['create_event'] | rand:3:8 (fail): tool_calls: one [], got ['create_event'] | rand:4:8 (fail): tool_calls: one [], got ['create_event'] | rand:5:8 (fail): tool_calls: one [], got ['create_event']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] create_event: arguments are not valid JSON text ('{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"]': Expecting ',' delimiter: line 1 column 72 (char 71))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] create_event: arguments are not valid JSON text ('{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"]': Expecting ',' delimiter: line 1 column 72 (char 71))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] create_event: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] create_event: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "llama3_json",
"reasoning_parser": null,
"parser_source": "docs/formats/llama.md (vLLM v0.30.0 row)",
"model_rule": "Llama-3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "unsloth/Llama-3.3-70B-Instruct",
"revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"requested_mode": "auto",
"resolved_revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "meta-llama/Llama-3.3-70B-Instruct",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|start_header_id|>assistant<|end_header_id|>\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "llama",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65"
},
"prompt_tail": "se the tools to answer.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"]"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"]"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "create_event",
"arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/vllm-v3-no-space",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": " {\"type\": \"number\"}}}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v3-arguments-before-name",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "us\", \"fahrenheit\"]}}}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v3-name-inside-arguments",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": " {\"type\": \"string\"}}}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v3-parallel",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "us\", \"fahrenheit\"]}}}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v3-content-before-call",
"family": "mistral",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello', got ' Hello'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "us\", \"fahrenheit\"]}}}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"streams": {
"one": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"special": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"token": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"rand:1:8": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"rand:2:8": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"rand:3:8": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"rand:4:8": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
},
"rand:5:8": {
"content": " Hello",
"reasoning_content": null,
"tool_calls": [
{
"name": "add",
"arguments_raw": "{\"a\": 1, \"b\": 2}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/vllm-v3-malformed-not-json",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'not json at all', got None"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "us\", \"fahrenheit\"]}}}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "not json at all",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/vllm-v11-single-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": ": {\"type\": \"number\"}}}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v11-parallel",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": ": {\"type\": \"number\"}}}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v11-content-before-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": ": {\"type\": \"number\"}}}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v13-think-no-end",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"city\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/vllm-v11-think-text",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2506",
"revision": "ad2fce5b4113139e1877dcadfd826c9262ad8e8c",
"requested_mode": "mistral",
"resolved_revision": "ad2fce5b4113139e1877dcadfd826c9262ad8e8c",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"city\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/llamacpp-ministral3-reasoning-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Reasoning-2512",
"revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630",
"requested_mode": "mistral",
"resolved_revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Reasoning-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"arg1\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/llamacpp-ministral3-parallel",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Reasoning-2512",
"revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630",
"requested_mode": "mistral",
"resolved_revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Reasoning-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"arg1\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/llamacpp-ministral3-marker-in-reasoning",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "content: expected None, got '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got 'Let me think about'; tool_calls: expected ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream None, got '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more'; tool_calls: nonstream ['special_function', 'special_function'], got ['special_function']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: nonstream '{\"arg1\":1}', got '{\"arg1\":1} and more[/THINK]'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: special (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function'] | token (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function'] | rand:1:8 (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function'] | rand:2:8 (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function'] | rand:3:8 (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function'] | rand:4:8 (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function'] | rand:5:8 (fail): content: one '[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: one ['special_function'], got ['special_function', 'special_function']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "fail",
"detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments contains '[/THINK]'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] special_function: arguments are not valid JSON text ('{\"arg1\":1} and more[/THINK]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] special_function: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Reasoning-2512",
"revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630",
"requested_mode": "mistral",
"resolved_revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Reasoning-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"arg1\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1}"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "[TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more",
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Let me think about",
"tool_calls": [
{
"name": "special_function",
"arguments_raw": "{\"arg1\":1} and more[/THINK]"
},
{
"name": "special_function",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/llamacpp-v11-call-id",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: special (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]'] | token (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]'] | rand:1:8 (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]'] | rand:2:8 (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]'] | rand:3:8 (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]'] | rand:4:8 (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]'] | rand:5:8 (fail): tool_calls: one ['special_function123456789'], got ['special_function123456789[ARGS]']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] special_function123456789[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[ARGS]': Extra data: line 1 column 12 (char 11))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'special_function123456789' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'special_function123456789' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] 'special_function123456789[ARGS]' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"arg1\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789",
"arguments_raw": "{\"arg1\": 1}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function123456789[ARGS]",
"arguments_raw": "{\"arg1\": 1}[ARGS]"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/llamacpp-v11-call-id-parallel",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001', 'special_function_with_opt000000002']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001', 'special_function_with_opt000000002']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: nonstream ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "fail",
"detail": "vs one: special (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', ''] | token (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', ''] | rand:1:8 (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', ''] | rand:2:8 (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', ''] | rand:3:8 (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', ''] | rand:4:8 (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', ''] | rand:5:8 (fail): tool_calls: one ['special_function000000001', 'special_function_with_opt000000002'], got ['special_function000000001[ARGS]', '']"
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].name contains '[ARGS]'; tool_calls[0].arguments contains '[TOOL_CALLS]'; tool_calls[0].arguments contains '[ARGS]'"
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] special_function000000001[ARGS]: arguments are not valid JSON text ('{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]': Extra data: line 1 column 12 (char 11)); [1] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'special_function000000001' is not an offered tool; [1] 'special_function_with_opt000000002' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] 'special_function000000001' is not an offered tool; [1] 'special_function_with_opt000000002' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] 'special_function000000001[ARGS]' is not an offered tool; [1] '' is not an offered tool"
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001', 'special_function_with_opt000000002']"
},
{
"check": "parallel_order",
"strategy": "one",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001', 'special_function_with_opt000000002']"
},
{
"check": "parallel_order",
"strategy": "special",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
},
{
"check": "parallel_order",
"strategy": "token",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "fail",
"detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 2 ['special_function000000001[ARGS]', '']"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"arg1\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001",
"arguments_raw": "{\"arg1\": 1}"
},
{
"name": "special_function_with_opt000000002",
"arguments_raw": "{\"arg1\": 1, \"arg2\": 2}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001",
"arguments_raw": "{\"arg1\": 1}"
},
{
"name": "special_function_with_opt000000002",
"arguments_raw": "{\"arg1\": 1, \"arg2\": 2}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": null,
"tool_calls": [
{
"name": "special_function000000001[ARGS]",
"arguments_raw": "{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}[ARGS]"
},
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/llamacpp-devstral-content-before-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Devstral-Small-2507",
"revision": "bd165ab26cebbcc2eea2c4ecbfc07f3ac42b3c39",
"requested_mode": "mistral",
"resolved_revision": "bd165ab26cebbcc2eea2c4ecbfc07f3ac42b3c39",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Devstral-Small-2507",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"arg1\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/bug-devstral-glob-braces-in-arguments",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Devstral-Small-2507",
"revision": "bd165ab26cebbcc2eea2c4ecbfc07f3ac42b3c39",
"requested_mode": "mistral",
"resolved_revision": "bd165ab26cebbcc2eea2c4ecbfc07f3ac42b3c39",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Devstral-Small-2507",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": "\"required\": [\"query\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-single-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-content-before-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-parallel-calls",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-no-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-empty-arguments",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-marker-in-arguments",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "reasoning_content: expected 'Document the format; [TOOL_CALLS] here is just text.', got 'Document the format; ...'; tool_calls: expected ['write_file'], got ['here is just text.[TOOL_CALLS]write_file', 'name']"
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "reasoning_content: nonstream 'Document the format; ...', got 'Document the format; [TOOL_CALLS] here is just text.'; tool_calls: nonstream ['here is just text.[TOOL_CALLS]write_file', 'name'], got ['write_file']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "fail",
"detail": "tool_calls[0].name contains '[TOOL_CALLS]'"
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] here is just text.[TOOL_CALLS]write_file: arguments are not valid JSON text ('{\"path\": \"fmt.md\", \"content\": \"Calls look like': Unterminated string starting at: line 1 column 31 (char 30)); [1] name: arguments are not valid JSON text ('{\\\\\"a\\\\\": 1}; thinking is .\"}': Expecting property name enclosed in double quotes: line 1 column 2 (char 1))"
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] 'here is just text.[TOOL_CALLS]write_file' is not an offered tool; [1] 'name' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "Document the format; ...",
"tool_calls": [
{
"name": "here is just text.[TOOL_CALLS]write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like"
},
{
"name": "name",
"arguments_raw": "{\\\"a\\\": 1}; thinking is .\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Document the format; [TOOL_CALLS] here is just text.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Calls look like [TOOL_CALLS]name[ARGS]{\\\"a\\\": 1}; thinking is [THINK]...[/THINK].\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/v13think-multiline-reasoning",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-truncated-in-reasoning",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13think-truncated-in-arguments",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside the arguments JSON; the call is incomplete."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /': Unterminated string starting at: line 1 column 38 (char 37))"
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] write_file: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"streams": {
"one": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"special": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"token": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"rand:1:8": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"rand:2:8": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"rand:3:8": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"rand:4:8": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
},
"rand:5:8": {
"content": null,
"reasoning_content": "Write the script.",
"tool_calls": [
{
"name": "write_file",
"arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/v13think-stop-at-open-marker",
"family": "mistral",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: expected [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls: nonstream [], got ['']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] : arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] '' is not an offered tool"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] '' is not an offered tool"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": "mistral",
"parser_source": "reasoning models ([THINK] v13+, v11): --reasoning-parser mistral; docs/formats/mistral.md",
"model_rule": "Magistral|Mistral-Medium-3\\.5|Mistral-Small-4|Ministral-3-.*Reasoning",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "mistralai/Magistral-Small-2509",
"revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"requested_mode": "mistral",
"resolved_revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Magistral-Small-2509",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"special": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"token": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check that for you.",
"reasoning_content": "The user wants the weather in Paris. I should call get_weather.",
"tool_calls": [
{
"name": "",
"arguments_raw": ""
}
],
"exception": null
}
}
}
},
{
"fixture_id": "mistral/v13-single-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-parallel-calls",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-content-before-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-no-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-nested-json",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-unicode-emoji",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-string-escapes",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-numeric-arguments",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-long-arguments",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-param-named-name",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v13-array-lookalike-argument",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Ministral-3-14B-Instruct-2512",
"revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"requested_mode": "mistral",
"resolved_revision": "29439f81c2be264d8d393273f99e7db9c0961120",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Ministral-3-14B-Instruct-2512",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v13"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v11-single-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v11-parallel-calls",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v11-content-before-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v11-no-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"requested_mode": "mistral",
"resolved_revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43",
"mode": "mistral",
"tokenizer_class": "vllm.tokenizers.mistral.MistralTokenizer"
},
"model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.SlowIncrementalDetokenizer",
"model_type": "mistral3",
"is_harmony": false,
"template": {
"source": "mistral_common via vLLM MistralTokenizer (no Jinja template)",
"mistral_common": "1.12.0",
"tokenizer_version": "v11"
},
"prompt_tail": " \"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST]Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v3-single-call",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "\"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v3-parallel-calls",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "parallel_order",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "\"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v3-empty-arguments",
"family": "mistral",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "\"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
}
},
{
"fixture_id": "mistral/v3-no-call",
"family": "mistral",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "mistral",
"reasoning_parser": null,
"parser_source": "Mistral model cards: --tool-call-parser mistral",
"model_rule": "mistral|ministral|devstral",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {},
"tokenizer": {
"repo": "mistralai/Mistral-7B-Instruct-v0.3",
"revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"requested_mode": "auto",
"resolved_revision": "c170c708c41dac9275d15a8fff4eca08d52bab71",
"mode": "hf",
"tokenizer_class": "transformers.tokenization_utils_tokenizers.TokenizersBackend"
},
"model": "mistralai/Mistral-7B-Instruct-v0.3",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "[/INST]",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "mistral",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826"
},
"prompt_tail": "\"required\": [\"name\"]}}}][/AVAILABLE_TOOLS][INST] Use the tools to answer.[/INST]",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": " The capital of France is Paris.",
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/single-call-thinking",
"family": "qwen3-hermes",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\""
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "end|>\n<|im_start|>user\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"special": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"token": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "\n\n",
"reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/single-call-no-thinking",
"family": "qwen3-hermes",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "qwen3-hermes/no-call-thinking",
"family": "qwen3-hermes",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "end|>\n<|im_start|>user\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"special": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"token": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "\n\nHello! How can I help you today?",
"reasoning_content": "\nThe user is just greeting me. No tool is needed.\n",
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/no-call-no-thinking",
"family": "qwen3-hermes",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "qwen3-hermes/text-before-call-thinking",
"family": "qwen3-hermes",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "end|>\n<|im_start|>user\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "\n\nLet me check the current conditions in London.\n",
"reasoning_content": "\nI need live weather data for London.\n",
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/text-before-call-no-thinking",
"family": "qwen3-hermes",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"special": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"token": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "Let me check the current conditions in London.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"London\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/empty-arguments-no-thinking",
"family": "qwen3-hermes",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "qwen3-hermes/empty-arguments-thinking",
"family": "qwen3-hermes",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "end|>\n<|im_start|>user\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"special": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"token": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:1:8": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:2:8": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:3:8": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:4:8": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
},
"rand:5:8": {
"content": "\n\n",
"reasoning_content": "\nget_time takes no arguments.\n",
"tool_calls": [
{
"name": "get_time",
"arguments_raw": "{}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-mid-arguments",
"family": "qwen3-hermes",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "special",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "token",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "fail",
"detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup."
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"San Fr': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "fail",
"detail": "[0] get_weather: arguments are not a JSON object; not validated"
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I will check.\n\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Fr",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"special": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"token": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"rand:1:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"rand:2:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"rand:3:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"rand:4:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
},
"rand:5:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [
{
"name": "get_weather",
"arguments_raw": "{\"city\": \"San Fr"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-at-opener",
"family": "qwen3-hermes",
"status": "fail",
"checks": [
{
"check": "expected_error",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_error",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"streams": {
"one": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"special": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"token": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:1:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:2:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:3:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:4:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
},
"rand:5:8": {
"content": "I will check.\n",
"reasoning_content": null,
"tool_calls": [],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/nested-json-deep",
"family": "qwen3-hermes",
"status": "pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": false,
"thinking": false
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n\n\n\n\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "ser\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"generation_prompt_match": true
}
},
{
"fixture_id": "qwen3-hermes/unicode-emoji-thinking",
"family": "qwen3-hermes",
"status": "soft_pass",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "soft_pass",
"detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "soft_pass",
"detail": "content (whitespace only): nonstream None, got '\\n\\n'"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "nonstream",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_schema",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
}
],
"parser_config": {
"engine": "vllm",
"pinned_version": "0.30.0",
"tool_parser": "hermes",
"reasoning_parser": "qwen3",
"parser_source": "Qwen3 model card: --tool-call-parser hermes --reasoning-parser qwen3",
"model_rule": "Qwen3",
"entrypoint": "vllm.parser.parser_manager.ParserManager.get_parser",
"enable_auto_tools": true,
"tool_choice": "auto (tools and tool_choice omitted when the fixture offers no tools)",
"chat_template_kwargs": {
"enable_thinking": true,
"thinking": true
},
"tokenizer": {
"repo": "Qwen/Qwen3-0.6B",
"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"requested_mode": "auto",
"resolved_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
"mode": "hf",
"tokenizer_class": "transformers.models.qwen2.tokenization_qwen2.Qwen2Tokenizer"
},
"model": "Qwen/Qwen3-0.6B",
"units_source": "fixture.output_token_ids",
"detokenizer": "vllm.v1.engine.detokenizer.IncrementalDetokenizer.from_new_request",
"stop_token_in_final_delta": false,
"prompt_messages": [
{
"role": "user",
"content": "Use the tools to answer."
}
],
"generation_prompt": "<|im_start|>assistant\n",
"detokenizer_class": "vllm.v1.engine.detokenizer.FastIncrementalDetokenizer",
"model_type": "qwen3",
"is_harmony": false,
"template": {
"source": "tokenizer chat template (HF repo)",
"sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8"
},
"prompt_tail": "end|>\n<|im_start|>user\nUse the tools to answer.<|im_end|>\n<|im_start|>assistant\n",
"generation_prompt_match": true
},
"observed": {
"nonstream": {
"content": null,
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"streams": {
"one": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"special": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"token": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:1:8": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:2:8": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:3:8": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:4:8": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
},
"rand:5:8": {
"content": "\n\n",
"reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n",
"tool_calls": [
{
"name": "translate",
"arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩💻\", \"target\": \"de\"}"
}
],
"exception": null
}
}
}
},
{
"fixture_id": "qwen3-hermes/marker-in-arguments",
"family": "qwen3-hermes",
"status": "fail",
"checks": [
{
"check": "expected_match",
"strategy": "nonstream",
"status": "fail",
"detail": "content: expected None, got '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls: expected ['echo'], got []"
},
{
"check": "expected_match",
"strategy": "one",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "special",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "token",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "rand:1:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "rand:2:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "rand:3:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "rand:4:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "expected_match",
"strategy": "rand:5:8",
"status": "fail",
"detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...'"
},
{
"check": "stream_equals_nonstream",
"strategy": "one",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "special",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "token",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:1:8",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:2:8",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:3:8",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:4:8",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "stream_equals_nonstream",
"strategy": "rand:5:8",
"status": "fail",
"detail": "content: nonstream '\\n{\"name\": \"echo\", \"arguments\": {\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}}\\n', got None; tool_calls: nonstream [], got ['echo']"
},
{
"check": "split_invariance",
"strategy": "*",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "nonstream",
"status": "fail",
"detail": "content contains ''; content contains ''; content contains ''"
},
{
"check": "no_leakage",
"strategy": "one",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "special",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "token",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:1:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:2:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:3:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:4:8",
"status": "pass",
"detail": null
},
{
"check": "no_leakage",
"strategy": "rand:5:8",
"status": "pass",
"detail": null
},
{
"check": "arguments_json",
"strategy": "one",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use ...': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "special",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use ...': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "token",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use ...': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:1:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use ...': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:2:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use ...': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:3:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use ...': Unterminated string starting at: line 1 column 10 (char 9))"
},
{
"check": "arguments_json",
"strategy": "rand:4:8",
"status": "fail",
"detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Use